@tinyshed/tinystore 0.1.0-rc.5 → 0.1.0-rc.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -37
- package/bin/tinystore.js +12 -1
- package/dist/backup.d.ts +33 -0
- package/dist/backup.js +49 -0
- package/dist/blobs.d.ts +160 -79
- package/dist/blobs.js +510 -217
- package/dist/channels.d.ts +90 -0
- package/dist/channels.js +291 -0
- package/dist/clock.js +3 -3
- package/dist/config.d.ts +209 -86
- package/dist/config.js +581 -538
- package/dist/connection.d.ts +64 -2
- package/dist/connection.js +201 -16
- package/dist/console.d.ts +27 -3
- package/dist/console.js +165 -12
- package/dist/dotenv.d.ts +7 -0
- package/dist/dotenv.js +79 -0
- package/dist/errors.d.ts +61 -1
- package/dist/errors.js +81 -1
- package/dist/handles.d.ts +44 -2
- package/dist/handles.js +81 -2
- package/dist/index.d.ts +17 -11
- package/dist/index.js +5 -3
- package/dist/instruments.d.ts +177 -0
- package/dist/instruments.js +819 -0
- package/dist/jobs.d.ts +335 -239
- package/dist/jobs.js +716 -536
- package/dist/json.d.ts +2 -0
- package/dist/json.js +122 -0
- package/dist/kv.d.ts +130 -243
- package/dist/kv.js +273 -640
- package/dist/limits.d.ts +80 -0
- package/dist/limits.js +139 -0
- package/dist/locks.d.ts +62 -0
- package/dist/locks.js +237 -0
- package/dist/logger.d.ts +45 -9
- package/dist/logger.js +150 -38
- package/dist/metrics.d.ts +210 -209
- package/dist/metrics.js +319 -556
- package/dist/once.d.ts +19 -18
- package/dist/once.js +74 -63
- package/dist/query.d.ts +181 -0
- package/dist/query.js +788 -0
- package/dist/records.d.ts +241 -150
- package/dist/records.js +507 -291
- package/dist/runtime/addon.d.ts +14 -0
- package/dist/runtime/addon.js +69 -0
- package/dist/runtime/bun.js +5 -7
- package/dist/runtime/node.js +5 -7
- package/dist/runtime/pipe.d.ts +17 -0
- package/dist/runtime/pipe.js +133 -0
- package/dist/runtime/shared.d.ts +6 -0
- package/dist/runtime/shared.js +19 -0
- package/dist/runtime.d.ts +5 -2
- package/dist/search.d.ts +189 -0
- package/dist/search.js +281 -0
- package/dist/session.d.ts +31 -3
- package/dist/session.js +153 -41
- package/dist/sql.d.ts +228 -78
- package/dist/sql.js +745 -240
- package/dist/store.d.ts +244 -24
- package/dist/store.js +270 -67
- package/dist/time.d.ts +5 -1
- package/dist/time.js +22 -2
- package/dist/tx.d.ts +61 -0
- package/dist/tx.js +314 -0
- package/dist/wire/codec.d.ts +23 -7
- package/dist/wire/codec.js +56 -35
- package/dist/wire/frame.d.ts +2 -0
- package/dist/wire/frame.js +31 -14
- package/dist/wire/messages.d.ts +259 -0
- package/dist/wire/messages.js +22 -0
- package/dist/wire/msgpack.d.ts +49 -1
- package/dist/wire/msgpack.js +293 -56
- package/dist/wire/protocol.d.ts +2683 -0
- package/dist/wire/protocol.js +10846 -0
- package/package.json +39 -38
- package/src/backup.ts +81 -0
- package/src/blobs.ts +678 -301
- package/src/channels.ts +377 -0
- package/src/clock.ts +4 -4
- package/src/config.ts +902 -612
- package/src/connection.ts +279 -21
- package/src/console.ts +180 -12
- package/src/dotenv.ts +82 -0
- package/src/errors.ts +105 -0
- package/src/handles.ts +157 -8
- package/src/index.ts +158 -52
- package/src/instruments.ts +1042 -0
- package/src/jobs.ts +1017 -733
- package/src/json.ts +133 -0
- package/src/kv.ts +424 -847
- package/src/limits.ts +202 -0
- package/src/locks.ts +320 -0
- package/src/logger.ts +209 -40
- package/src/metrics.ts +557 -752
- package/src/once.ts +86 -83
- package/src/query.ts +1005 -0
- package/src/records.ts +795 -418
- package/src/runtime/addon.ts +99 -0
- package/src/runtime/bun.ts +5 -7
- package/src/runtime/node.ts +5 -7
- package/src/runtime/pipe.ts +178 -0
- package/src/runtime/shared.ts +20 -0
- package/src/runtime.ts +5 -2
- package/src/search.ts +480 -0
- package/src/session.ts +187 -41
- package/src/sql.ts +993 -306
- package/src/store.ts +473 -78
- package/src/time.ts +48 -3
- package/src/tx.ts +426 -0
- package/src/wire/codec.ts +78 -37
- package/src/wire/frame.ts +44 -14
- package/src/wire/messages.ts +25 -0
- package/src/wire/msgpack.ts +327 -56
- package/src/wire/protocol.ts +11709 -0
- package/dist/limiter.d.ts +0 -45
- package/dist/limiter.js +0 -68
- package/dist/quota.d.ts +0 -44
- package/dist/quota.js +0 -98
- package/src/limiter.ts +0 -99
- package/src/quota.ts +0 -137
package/src/blobs.ts
CHANGED
|
@@ -1,238 +1,248 @@
|
|
|
1
|
-
// The application's files
|
|
2
|
-
//
|
|
3
|
-
//
|
|
1
|
+
// The application's files, as docs/guides/blobs/README.md has them: files by path in
|
|
2
|
+
// named sets, written whole and replaced whole, over the protocol. A file's
|
|
3
|
+
// bytes go in pieces within the credit of the side that takes them, and a
|
|
4
|
+
// whole read ends CorruptError before its last piece when its bytes changed.
|
|
4
5
|
|
|
5
|
-
import { type Connection,
|
|
6
|
+
import { type Connection, type Idempotence, type Link, pieceOf } from './connection.ts'
|
|
6
7
|
import { ClosedError, InvalidError, OutcomeUnknownError } from './errors.ts'
|
|
7
|
-
import { checkName, handleOn, type
|
|
8
|
-
import {
|
|
9
|
-
import { type Duration, dateOf, ms,
|
|
10
|
-
import type { Key } from './wire/codec.ts'
|
|
8
|
+
import { checkName, handleOn, type Via, viaLink } from './handles.ts'
|
|
9
|
+
import { ConnectionLostError, type Stream, watch } from './session.ts'
|
|
10
|
+
import { type Duration, dateOf, ms, unixMs } from './time.ts'
|
|
11
11
|
import {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
12
|
+
BlobsAt,
|
|
13
|
+
BlobsDelete,
|
|
14
|
+
BlobsExpire,
|
|
15
|
+
BlobsFolder,
|
|
16
|
+
BlobsFound,
|
|
17
|
+
BlobsGot,
|
|
18
|
+
BlobsHead,
|
|
19
|
+
BlobsInfo,
|
|
20
|
+
BlobsList,
|
|
21
|
+
BlobsMove,
|
|
22
|
+
BlobsOpen,
|
|
15
23
|
BlobsPage,
|
|
16
|
-
|
|
24
|
+
BlobsPiece,
|
|
25
|
+
BlobsRead,
|
|
26
|
+
BlobsUsage,
|
|
27
|
+
BlobsWrite,
|
|
28
|
+
BlobsWritten,
|
|
17
29
|
methods,
|
|
18
|
-
} from './wire/
|
|
30
|
+
} from './wire/protocol.ts'
|
|
19
31
|
|
|
20
|
-
/**
|
|
21
|
-
|
|
32
|
+
/** A size in bytes: a number, or text in binary units, 64KiB or 2GiB; '2GB' does not compile. */
|
|
33
|
+
export type Size = number | `${number}${'KiB' | 'MiB' | 'GiB' | 'TiB'}`
|
|
22
34
|
|
|
23
|
-
export interface
|
|
24
|
-
/**
|
|
25
|
-
|
|
35
|
+
export interface FilesOptions {
|
|
36
|
+
/** the term of every file written without its own; not a ceiling */
|
|
37
|
+
ttl?: Duration | undefined
|
|
38
|
+
/** the largest file a write may leave: past it, a write is refused and leaves nothing */
|
|
39
|
+
maxFileSize?: Size | undefined
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** What a file carries: what serving it needs. */
|
|
43
|
+
export interface FileInfo {
|
|
44
|
+
/** its path within the folder it was asked from */
|
|
45
|
+
path: string
|
|
26
46
|
size: number
|
|
27
|
-
/**
|
|
47
|
+
/** the SHA-256 of its bytes in hex, quoted, as an HTTP ETag is */
|
|
28
48
|
etag: string
|
|
29
|
-
contentType: string
|
|
30
|
-
|
|
31
|
-
/** undefined for
|
|
49
|
+
contentType: string
|
|
50
|
+
lastModified: Date
|
|
51
|
+
/** undefined for a file that does not expire */
|
|
32
52
|
expires: Date | undefined
|
|
33
53
|
meta: Record<string, string>
|
|
34
54
|
}
|
|
35
55
|
|
|
36
|
-
export interface
|
|
37
|
-
contentType?: string
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
ttl?: Duration
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
56
|
+
export interface FileOptions {
|
|
57
|
+
contentType?: string | undefined
|
|
58
|
+
meta?: Record<string, string> | undefined
|
|
59
|
+
/** the file's own term, from this write */
|
|
60
|
+
ttl?: Duration | undefined
|
|
61
|
+
/** the ETag of the one file this write may replace: another is ConflictError */
|
|
62
|
+
ifMatch?: string | undefined
|
|
63
|
+
/** the body's length: one longer or shorter is refused and leaves nothing */
|
|
64
|
+
size?: number | undefined
|
|
65
|
+
signal?: AbortSignal | undefined
|
|
46
66
|
}
|
|
47
67
|
|
|
48
|
-
|
|
49
|
-
/** the body's length, when it cannot say itself: one that disagrees leaves nothing */
|
|
50
|
-
size?: number
|
|
51
|
-
signal?: AbortSignal
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
/** What a put takes: bytes, text, a Blob or a file, or a stream of bytes. */
|
|
68
|
+
/** What a write takes: text, bytes, a Blob or a file, or a stream of bytes. */
|
|
55
69
|
export type Body =
|
|
70
|
+
| string
|
|
56
71
|
| Uint8Array
|
|
57
72
|
| ArrayBuffer
|
|
58
|
-
| string
|
|
59
73
|
| Blob
|
|
60
74
|
| ReadableStream<Uint8Array>
|
|
61
75
|
| AsyncIterable<Uint8Array>
|
|
62
76
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
77
|
+
export interface FileListOptions {
|
|
78
|
+
/** text within the folder: 'photos/1' finds photos/10.jpg too */
|
|
79
|
+
prefix?: string | undefined
|
|
80
|
+
/** a page's next */
|
|
81
|
+
after?: string | undefined
|
|
82
|
+
/** 1000 at most, and when absent */
|
|
83
|
+
limit?: number | undefined
|
|
70
84
|
}
|
|
71
85
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
bucket(name: string, options?: { defaultTtl?: Duration; maxSize?: number }): BlobBucket {
|
|
80
|
-
checkName(name, 'bucket')
|
|
81
|
-
const open = BlobsBucket.encode({
|
|
82
|
-
name,
|
|
83
|
-
defaultTtl: options?.defaultTtl === undefined ? undefined : ms(options.defaultTtl),
|
|
84
|
-
maxSize: options?.maxSize,
|
|
85
|
-
})
|
|
86
|
-
return new BlobBucket(this.#link, name, open, [])
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
function ownerText(owner: Key): string | Uint8Array {
|
|
91
|
-
if (typeof owner === 'number' || typeof owner === 'bigint') {
|
|
92
|
-
if (typeof owner === 'number' && !Number.isSafeInteger(owner)) {
|
|
93
|
-
throw new InvalidError(`the owner ${owner} is not an integer; an owner is text or an integer`)
|
|
94
|
-
}
|
|
95
|
-
return String(owner)
|
|
96
|
-
}
|
|
97
|
-
return owner
|
|
86
|
+
/** A page of a folder's files in the byte order of their paths, and where the next starts. */
|
|
87
|
+
export interface FilePage {
|
|
88
|
+
files: FileInfo[]
|
|
89
|
+
/** undefined on the last page */
|
|
90
|
+
next: string | undefined
|
|
98
91
|
}
|
|
99
92
|
|
|
100
|
-
const
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
const meta: Record<string, string> = {}
|
|
104
|
-
for (const [name, value] of Object.entries(o.meta ?? {})) {
|
|
105
|
-
meta[name] = typeof value === 'string' ? value : lossy.decode(value)
|
|
106
|
-
}
|
|
107
|
-
return {
|
|
108
|
-
key: o.key ?? '',
|
|
109
|
-
size: o.size ?? 0,
|
|
110
|
-
etag: o.etag ?? '',
|
|
111
|
-
contentType: o.contentType,
|
|
112
|
-
modified: new Date(o.modified ?? 0),
|
|
113
|
-
expires: dateOf(o.expires),
|
|
114
|
-
meta,
|
|
115
|
-
}
|
|
116
|
-
}
|
|
93
|
+
const encoder = new TextEncoder()
|
|
94
|
+
const decoder = new TextDecoder()
|
|
95
|
+
const nothing = new Uint8Array(0)
|
|
117
96
|
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
97
|
+
/** Files as `store.files` opens them. */
|
|
98
|
+
export function openFiles(link: Link, name: string, options?: FilesOptions): Files {
|
|
99
|
+
checkName(name, 'files')
|
|
100
|
+
const open = BlobsOpen.encode({
|
|
101
|
+
name,
|
|
122
102
|
ttl: options?.ttl === undefined ? undefined : ms(options.ttl),
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
}
|
|
103
|
+
maxFileSize: options?.maxFileSize === undefined ? undefined : bytesOf(options.maxFileSize),
|
|
104
|
+
})
|
|
105
|
+
return new Files(link, name, open, [])
|
|
127
106
|
}
|
|
128
107
|
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
108
|
+
/**
|
|
109
|
+
* A named set of files by path, or a folder of it, whose calls are the
|
|
110
|
+
* folder's: a path in it is the folder's segments and then its own.
|
|
111
|
+
*/
|
|
112
|
+
export class Files {
|
|
133
113
|
readonly name: string
|
|
134
114
|
readonly #link: Link
|
|
115
|
+
readonly #via: Via
|
|
135
116
|
readonly #open: Uint8Array
|
|
136
|
-
readonly #
|
|
117
|
+
readonly #folder: string[]
|
|
137
118
|
|
|
138
|
-
|
|
139
|
-
|
|
119
|
+
/** Files come from `store.files`. */
|
|
120
|
+
constructor(link: Link, name: string, open: Uint8Array, folder: string[]) {
|
|
140
121
|
this.name = name
|
|
122
|
+
this.#link = link
|
|
123
|
+
this.#via = viaLink(link)
|
|
141
124
|
this.#open = open
|
|
142
|
-
this.#
|
|
125
|
+
this.#folder = folder
|
|
143
126
|
}
|
|
144
127
|
|
|
145
|
-
/** The folder
|
|
146
|
-
|
|
147
|
-
return new
|
|
148
|
-
...this.#
|
|
149
|
-
...
|
|
128
|
+
/** The folder of these segments, one each: `folder('users', 42)` is users/42, never users/420. */
|
|
129
|
+
folder(...segments: (string | number | bigint)[]): Files {
|
|
130
|
+
return new Files(this.#link, this.name, this.#open, [
|
|
131
|
+
...this.#folder,
|
|
132
|
+
...segments.map(segmentOf),
|
|
150
133
|
])
|
|
151
134
|
}
|
|
152
135
|
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
136
|
+
/**
|
|
137
|
+
* Writes the file whole and replaces whatever the path held, returning
|
|
138
|
+
* once it is on disk as the store's durability says. Nobody sees half of
|
|
139
|
+
* it, while it is written or after a crash.
|
|
140
|
+
*/
|
|
141
|
+
async put(path: string, body: Body, options?: FileOptions): Promise<FileInfo> {
|
|
142
|
+
// a put that returned wrote: only a create writes nothing
|
|
143
|
+
return (await this.#write(path, body, options, false)) as FileInfo
|
|
160
144
|
}
|
|
161
145
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
)
|
|
146
|
+
/** Writes the file only where the path holds none, and says whether it did. */
|
|
147
|
+
async create(path: string, body: Body, options?: Omit<FileOptions, 'ifMatch'>): Promise<boolean> {
|
|
148
|
+
return (await this.#write(path, body, options, true)) !== undefined
|
|
166
149
|
}
|
|
167
150
|
|
|
168
|
-
/**
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
151
|
+
/**
|
|
152
|
+
* What the file carries, and its bytes: a file up to about a megabyte comes
|
|
153
|
+
* with them, and a larger one's are read when they are read, on its ETag,
|
|
154
|
+
* as S3's GET with If-Match reads them. Undefined when the path holds none.
|
|
155
|
+
*/
|
|
156
|
+
async get(path: string): Promise<StoredFile | undefined> {
|
|
157
|
+
const answer = await this.#call(
|
|
158
|
+
'blobs.get',
|
|
159
|
+
handle => BlobsAt.encode({ handle, folder: this.#folder, path }),
|
|
160
|
+
'read',
|
|
161
|
+
)
|
|
162
|
+
const got = BlobsGot.decode(answer)
|
|
163
|
+
if (got.info === undefined) {
|
|
164
|
+
return undefined
|
|
165
|
+
}
|
|
166
|
+
const info = infoOf(got.info)
|
|
167
|
+
const bytes = got.bytes ?? nothing
|
|
168
|
+
const whole = bytes.length === info.size ? bytes : undefined
|
|
169
|
+
return new StoredFile(info, whole, (offset, length) =>
|
|
170
|
+
this.#read(path, info.etag, offset, length),
|
|
171
|
+
)
|
|
172
172
|
}
|
|
173
173
|
|
|
174
|
-
|
|
175
|
-
|
|
174
|
+
/** What the file carries and none of its bytes; undefined when the path holds none. */
|
|
175
|
+
async head(path: string): Promise<FileInfo | undefined> {
|
|
176
|
+
const answer = await this.#call(
|
|
177
|
+
'blobs.head',
|
|
178
|
+
handle => BlobsAt.encode({ handle, folder: this.#folder, path }),
|
|
179
|
+
'read',
|
|
180
|
+
)
|
|
181
|
+
const head = BlobsHead.decode(answer)
|
|
182
|
+
return head.info === undefined ? undefined : infoOf(head.info)
|
|
176
183
|
}
|
|
177
184
|
|
|
178
|
-
/**
|
|
179
|
-
async
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
185
|
+
/** Deletes the file, or with ifMatch only that version of it; no file there is not an error. */
|
|
186
|
+
async delete(path: string, options?: { ifMatch?: string | undefined }): Promise<void> {
|
|
187
|
+
await this.#call(
|
|
188
|
+
'blobs.delete',
|
|
189
|
+
handle =>
|
|
190
|
+
BlobsDelete.encode({ handle, folder: this.#folder, path, ifMatch: options?.ifMatch }),
|
|
191
|
+
'write',
|
|
184
192
|
)
|
|
185
193
|
}
|
|
186
194
|
|
|
187
|
-
/**
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
)
|
|
195
|
+
/**
|
|
196
|
+
* A copy at `to` that shares the bytes, which are not copied. A file at
|
|
197
|
+
* `to` is ConflictError unless ifMatch names the version to replace.
|
|
198
|
+
*/
|
|
199
|
+
copy(from: string, to: string, options?: { ifMatch?: string | undefined }): Promise<FileInfo> {
|
|
200
|
+
return this.#move('blobs.copy', from, to, options?.ifMatch)
|
|
194
201
|
}
|
|
195
202
|
|
|
196
|
-
/**
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
203
|
+
/**
|
|
204
|
+
* The file moved to `to` in one write: a file there is ConflictError
|
|
205
|
+
* unless ifMatch names the version to replace, and none at `from` NotFoundError.
|
|
206
|
+
*/
|
|
207
|
+
rename(from: string, to: string, options?: { ifMatch?: string | undefined }): Promise<FileInfo> {
|
|
208
|
+
return this.#move('blobs.rename', from, to, options?.ifMatch)
|
|
200
209
|
}
|
|
201
210
|
|
|
202
|
-
/**
|
|
203
|
-
async
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
211
|
+
/** Sets when the file expires, a span from now or a moment; says whether there was a file. */
|
|
212
|
+
async expire(path: string, after: Duration | Date): Promise<boolean> {
|
|
213
|
+
const span = after instanceof Date ? Math.max(0, unixMs(after) - Date.now()) : ms(after)
|
|
214
|
+
const answer = await this.#call(
|
|
215
|
+
'blobs.expire',
|
|
216
|
+
handle => BlobsExpire.encode({ handle, folder: this.#folder, path, after: span }),
|
|
217
|
+
'write',
|
|
218
|
+
)
|
|
219
|
+
return BlobsFound.decode(answer).found === true
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/** A page of the files in the folder and in the folders in it, in the byte order of their paths. */
|
|
223
|
+
async list(options?: FileListOptions): Promise<FilePage> {
|
|
224
|
+
const answer = await this.#call(
|
|
225
|
+
'blobs.list',
|
|
226
|
+
handle =>
|
|
227
|
+
BlobsList.encode({
|
|
228
|
+
handle,
|
|
229
|
+
folder: this.#folder,
|
|
230
|
+
prefix: options?.prefix ?? '',
|
|
219
231
|
after: options?.after,
|
|
220
232
|
limit: options?.limit,
|
|
221
233
|
}),
|
|
222
|
-
|
|
234
|
+
'read',
|
|
223
235
|
)
|
|
224
|
-
const page = BlobsPage.decode(
|
|
225
|
-
return {
|
|
226
|
-
items: got.items.map(item => objectOf(BlobsObject.decode(item))),
|
|
227
|
-
next: page.more === true ? (page.after ?? '') : undefined,
|
|
228
|
-
}
|
|
236
|
+
const page = BlobsPage.decode(answer)
|
|
237
|
+
return { files: (page.files ?? []).map(infoOf), next: page.next }
|
|
229
238
|
}
|
|
230
239
|
|
|
231
|
-
|
|
240
|
+
/** Every file of the folder, a page at a time, holding nothing between pages. */
|
|
241
|
+
async *all(options?: { prefix?: string | undefined }): AsyncGenerator<FileInfo> {
|
|
232
242
|
let after: string | undefined
|
|
233
243
|
for (;;) {
|
|
234
|
-
const page = await this.
|
|
235
|
-
yield* page.
|
|
244
|
+
const page = await this.list({ prefix: options?.prefix, after })
|
|
245
|
+
yield* page.files
|
|
236
246
|
if (page.next === undefined) {
|
|
237
247
|
return
|
|
238
248
|
}
|
|
@@ -240,198 +250,565 @@ export class BlobBucket {
|
|
|
240
250
|
}
|
|
241
251
|
}
|
|
242
252
|
|
|
253
|
+
/** The files in the folder and their bytes, as it is now: it counts, it does not limit. */
|
|
254
|
+
async usage(): Promise<{ count: number; size: number }> {
|
|
255
|
+
const answer = await this.#call(
|
|
256
|
+
'blobs.usage',
|
|
257
|
+
handle => BlobsFolder.encode({ handle, folder: this.#folder }),
|
|
258
|
+
'read',
|
|
259
|
+
)
|
|
260
|
+
const usage = BlobsUsage.decode(answer)
|
|
261
|
+
return { count: usage.count ?? 0, size: usage.size ?? 0 }
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/** Removes the folder's every file and every folder in it, at once for every reader. */
|
|
265
|
+
async clear(): Promise<void> {
|
|
266
|
+
await this.#call(
|
|
267
|
+
'blobs.clear',
|
|
268
|
+
handle => BlobsFolder.encode({ handle, folder: this.#folder }),
|
|
269
|
+
'write',
|
|
270
|
+
)
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Begins a file written a piece at a time, which appears at `commit` and
|
|
275
|
+
* never before: `await using upload = await files.upload(path)`, and
|
|
276
|
+
* leaving the block without commit aborts it.
|
|
277
|
+
*/
|
|
278
|
+
async upload(path: string, options?: FileOptions): Promise<Upload> {
|
|
279
|
+
const connection = await this.#link.connection()
|
|
280
|
+
const write = BlobsWrite.encode({
|
|
281
|
+
...this.#fields(path, options),
|
|
282
|
+
handle: await this.#handle(connection),
|
|
283
|
+
})
|
|
284
|
+
const stream = await connection.session.open(methods['blobs.upload'], write, false)
|
|
285
|
+
return new Upload(stream, pieceOf(connection), options?.signal)
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
#handle(connection: Connection): Promise<number> {
|
|
289
|
+
return handleOn(connection, methods['blobs.open'], this.#open)
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
#call(
|
|
293
|
+
method: keyof typeof methods,
|
|
294
|
+
body: (handle: number) => Uint8Array,
|
|
295
|
+
idempotence: Idempotence,
|
|
296
|
+
): Promise<Uint8Array> {
|
|
297
|
+
return this.#via.call(methods['blobs.open'], this.#open, methods[method], body, idempotence)
|
|
298
|
+
}
|
|
299
|
+
|
|
243
300
|
/**
|
|
244
|
-
*
|
|
245
|
-
*
|
|
246
|
-
*
|
|
301
|
+
* A file's bytes as S3's GET reads them with a range and If-Match: on the
|
|
302
|
+
* ETag a get gave, ConflictError once the path holds another file or none;
|
|
303
|
+
* the whole file checked, a range not. Nothing is asked before the first read.
|
|
247
304
|
*/
|
|
248
|
-
|
|
249
|
-
|
|
305
|
+
#read(
|
|
306
|
+
path: string,
|
|
307
|
+
ifMatch: string,
|
|
308
|
+
offset?: number,
|
|
309
|
+
length?: number,
|
|
310
|
+
): ReadableStream<Uint8Array> {
|
|
311
|
+
let stream: Stream | undefined
|
|
312
|
+
const open = () =>
|
|
313
|
+
this.#link.run('read', async connection => {
|
|
314
|
+
const read = BlobsRead.encode({
|
|
315
|
+
handle: await this.#handle(connection),
|
|
316
|
+
folder: this.#folder,
|
|
317
|
+
path,
|
|
318
|
+
ifMatch,
|
|
319
|
+
offset,
|
|
320
|
+
length,
|
|
321
|
+
})
|
|
322
|
+
const opened = await connection.session.open(methods['blobs.read'], read, true)
|
|
323
|
+
return { opened, first: await opened.next() }
|
|
324
|
+
})
|
|
325
|
+
return new ReadableStream<Uint8Array>({
|
|
326
|
+
async pull(controller) {
|
|
327
|
+
try {
|
|
328
|
+
if (stream === undefined) {
|
|
329
|
+
// the RESPONSE's piece is no DATA, so it takes none of the credit
|
|
330
|
+
const { opened, first } = await open()
|
|
331
|
+
stream = opened
|
|
332
|
+
enqueue(controller, first.body)
|
|
333
|
+
if (first.end) {
|
|
334
|
+
controller.close()
|
|
335
|
+
}
|
|
336
|
+
return
|
|
337
|
+
}
|
|
338
|
+
const event = await stream.next()
|
|
339
|
+
if (event.end) {
|
|
340
|
+
controller.close()
|
|
341
|
+
return
|
|
342
|
+
}
|
|
343
|
+
stream.consumed(event.body.length)
|
|
344
|
+
enqueue(controller, event.body)
|
|
345
|
+
} catch (err) {
|
|
346
|
+
controller.error(err)
|
|
347
|
+
}
|
|
348
|
+
},
|
|
349
|
+
cancel() {
|
|
350
|
+
stream?.cancel()
|
|
351
|
+
},
|
|
352
|
+
})
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
async #move(
|
|
356
|
+
method: 'blobs.copy' | 'blobs.rename',
|
|
357
|
+
from: string,
|
|
358
|
+
to: string,
|
|
359
|
+
ifMatch: string | undefined,
|
|
360
|
+
): Promise<FileInfo> {
|
|
361
|
+
const answer = await this.#call(
|
|
362
|
+
method,
|
|
363
|
+
handle => BlobsMove.encode({ handle, folder: this.#folder, from, to, ifMatch }),
|
|
364
|
+
'write',
|
|
365
|
+
)
|
|
366
|
+
return infoOf(BlobsInfo.decode(answer))
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
#fields(path: string, options: FileOptions | undefined) {
|
|
370
|
+
return {
|
|
371
|
+
folder: this.#folder,
|
|
372
|
+
path,
|
|
373
|
+
contentType: options?.contentType,
|
|
374
|
+
meta: options?.meta,
|
|
375
|
+
ttl: options?.ttl === undefined ? undefined : ms(options.ttl),
|
|
376
|
+
ifMatch: options?.ifMatch,
|
|
377
|
+
size: options?.size,
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* Writes a body: in one call when it fits a piece, else as an upload over
|
|
383
|
+
* a stream. What it wrote, nothing when a create found a file.
|
|
384
|
+
*/
|
|
385
|
+
async #write(
|
|
386
|
+
path: string,
|
|
387
|
+
body: Body,
|
|
388
|
+
options: FileOptions | undefined,
|
|
389
|
+
create: boolean,
|
|
390
|
+
): Promise<FileInfo | undefined> {
|
|
391
|
+
const source = sourceOf(body)
|
|
250
392
|
const fields = {
|
|
251
|
-
...
|
|
393
|
+
...this.#fields(path, options),
|
|
252
394
|
contentType: options?.contentType ?? source.contentType,
|
|
395
|
+
size: options?.size ?? source.size,
|
|
396
|
+
create,
|
|
253
397
|
}
|
|
254
398
|
return this.#link.run(
|
|
255
399
|
'write',
|
|
256
400
|
async connection => {
|
|
257
|
-
const
|
|
258
|
-
|
|
401
|
+
const handle = await this.#handle(connection)
|
|
402
|
+
const piece = pieceOf(connection)
|
|
403
|
+
const whole = await source.whole(piece)
|
|
404
|
+
if (whole !== undefined) {
|
|
405
|
+
const write = BlobsWrite.encode({ ...fields, handle, bytes: whole })
|
|
406
|
+
return writtenOf(await connection.session.call(methods['blobs.put'], write))
|
|
407
|
+
}
|
|
408
|
+
const write = BlobsWrite.encode({ ...fields, handle })
|
|
409
|
+
const stream = await connection.session.open(methods['blobs.upload'], write, false)
|
|
410
|
+
return uploaded(stream, source, piece, options?.signal)
|
|
259
411
|
},
|
|
260
412
|
options?.signal,
|
|
261
413
|
)
|
|
262
414
|
}
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
/**
|
|
418
|
+
* A file written a piece at a time, as `files.upload` begins it: bytes go in
|
|
419
|
+
* with `write`, the file appears at `commit` and never before, and an upload
|
|
420
|
+
* that ends otherwise leaves nothing.
|
|
421
|
+
*/
|
|
422
|
+
export class Upload implements AsyncDisposable {
|
|
423
|
+
readonly #stream: Stream
|
|
424
|
+
readonly #piece: Uint8Array
|
|
425
|
+
readonly #unwatch: () => void
|
|
426
|
+
#filled = 0
|
|
427
|
+
/** the writes so far, one after another, so that two writes never interleave */
|
|
428
|
+
#writing: Promise<void> = Promise.resolve()
|
|
429
|
+
#done = false
|
|
430
|
+
|
|
431
|
+
/** Uploads come from `files.upload`. */
|
|
432
|
+
constructor(stream: Stream, piece: number, signal: AbortSignal | undefined) {
|
|
433
|
+
this.#stream = stream
|
|
434
|
+
this.#piece = new Uint8Array(piece)
|
|
435
|
+
this.#unwatch = watch(signal, stream)
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
/** Takes bytes, text as UTF-8, waiting while the server catches up. */
|
|
439
|
+
write(bytes: Uint8Array | string): Promise<void> {
|
|
440
|
+
if (this.#done) {
|
|
441
|
+
return Promise.reject(new InvalidError('the upload has ended'))
|
|
442
|
+
}
|
|
443
|
+
const written = this.#writing.then(() => this.#take(bytes))
|
|
444
|
+
this.#writing = written.catch(() => {})
|
|
445
|
+
return written
|
|
446
|
+
}
|
|
263
447
|
|
|
264
448
|
/**
|
|
265
|
-
*
|
|
266
|
-
*
|
|
267
|
-
* long, is not checked against the object's hash.
|
|
449
|
+
* Publishes the file whole, replacing what the path held as `put` does,
|
|
450
|
+
* once it is on disk as the store's durability says.
|
|
268
451
|
*/
|
|
269
|
-
async
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
452
|
+
async commit(): Promise<FileInfo> {
|
|
453
|
+
if (this.#done) {
|
|
454
|
+
throw new InvalidError('the upload has ended')
|
|
455
|
+
}
|
|
456
|
+
this.#done = true
|
|
457
|
+
let ending = false
|
|
458
|
+
try {
|
|
459
|
+
await this.#writing
|
|
460
|
+
ending = true
|
|
461
|
+
await this.#send(true)
|
|
462
|
+
// the RESPONSE that began it, and then the file the commit wrote
|
|
463
|
+
await this.#stream.next()
|
|
464
|
+
return writtenOf((await this.#stream.next()).body) as FileInfo
|
|
465
|
+
} catch (err) {
|
|
466
|
+
throw lostAs(err, ending)
|
|
467
|
+
} finally {
|
|
468
|
+
this.#unwatch()
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
/** Ends the upload, which leaves nothing; after `commit` it does nothing. */
|
|
473
|
+
async abort(): Promise<void> {
|
|
474
|
+
if (this.#done) {
|
|
475
|
+
return
|
|
476
|
+
}
|
|
477
|
+
this.#done = true
|
|
478
|
+
this.#stream.cancel()
|
|
479
|
+
this.#unwatch()
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
async [Symbol.asyncDispose](): Promise<void> {
|
|
483
|
+
await this.abort()
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
async #take(bytes: Uint8Array | string): Promise<void> {
|
|
487
|
+
const chunk = typeof bytes === 'string' ? encoder.encode(bytes) : bytes
|
|
488
|
+
try {
|
|
489
|
+
for (let at = 0; at < chunk.length; ) {
|
|
490
|
+
const taken = Math.min(this.#piece.length - this.#filled, chunk.length - at)
|
|
491
|
+
this.#piece.set(chunk.subarray(at, at + taken), this.#filled)
|
|
492
|
+
this.#filled += taken
|
|
493
|
+
at += taken
|
|
494
|
+
if (this.#filled === this.#piece.length) {
|
|
495
|
+
await this.#send(false)
|
|
291
496
|
}
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
497
|
+
}
|
|
498
|
+
} catch (err) {
|
|
499
|
+
throw lostAs(err, false)
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
/** Sends what the piece holds, the message a copy of it, so the piece fills again at once. */
|
|
504
|
+
async #send(end: boolean): Promise<void> {
|
|
505
|
+
const body = BlobsPiece.encode({ bytes: this.#piece.subarray(0, this.#filled) })
|
|
506
|
+
this.#filled = 0
|
|
507
|
+
await this.#stream.send(body, end)
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
/**
|
|
512
|
+
* Bytes of a file, read when they are read, as a Blob's are: `stream()`, or
|
|
513
|
+
* all of them with `bytes()`, `text()`, `json()` or `arrayBuffer()`, which
|
|
514
|
+
* hold them in memory. Each read is a read of its own.
|
|
515
|
+
*/
|
|
516
|
+
export class FileBytes {
|
|
517
|
+
readonly size: number
|
|
518
|
+
readonly #stream: () => ReadableStream<Uint8Array>
|
|
519
|
+
|
|
520
|
+
/** File bytes come from `files.get` and `slice`. */
|
|
521
|
+
constructor(size: number, stream: () => ReadableStream<Uint8Array>) {
|
|
522
|
+
this.size = size
|
|
523
|
+
this.#stream = stream
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
/** Its bytes as they come, the credit of each given back once it is taken. */
|
|
527
|
+
stream(): ReadableStream<Uint8Array> {
|
|
528
|
+
return this.#stream()
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
async bytes(): Promise<Uint8Array<ArrayBuffer>> {
|
|
532
|
+
const whole = new Uint8Array(this.size)
|
|
533
|
+
let at = 0
|
|
534
|
+
for await (const piece of this.stream() as AsyncIterable<Uint8Array>) {
|
|
535
|
+
whole.set(piece, at)
|
|
536
|
+
at += piece.length
|
|
537
|
+
}
|
|
538
|
+
return at === whole.length ? whole : whole.slice(0, at)
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
async arrayBuffer(): Promise<ArrayBuffer> {
|
|
542
|
+
return (await this.bytes()).buffer
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
async text(): Promise<string> {
|
|
546
|
+
return decoder.decode(await this.bytes())
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
async json(): Promise<unknown> {
|
|
550
|
+
return JSON.parse(await this.text())
|
|
296
551
|
}
|
|
297
552
|
}
|
|
298
553
|
|
|
299
554
|
/**
|
|
300
|
-
* A
|
|
301
|
-
*
|
|
555
|
+
* A file as `get` found it: what it carries, and its bytes. A whole read is
|
|
556
|
+
* checked against its SHA-256 and fails CorruptError before its last bytes
|
|
557
|
+
* when they changed; `slice` reads a range, which is not checked.
|
|
558
|
+
*/
|
|
559
|
+
export class StoredFile extends FileBytes implements FileInfo {
|
|
560
|
+
readonly path: string
|
|
561
|
+
readonly etag: string
|
|
562
|
+
readonly contentType: string
|
|
563
|
+
readonly lastModified: Date
|
|
564
|
+
readonly expires: Date | undefined
|
|
565
|
+
readonly meta: Record<string, string>
|
|
566
|
+
/** its bytes when they came with the get */
|
|
567
|
+
readonly #whole: Uint8Array | undefined
|
|
568
|
+
readonly #read: (offset?: number, length?: number) => ReadableStream<Uint8Array>
|
|
569
|
+
|
|
570
|
+
/** Stored files come from `files.get`. */
|
|
571
|
+
constructor(
|
|
572
|
+
info: FileInfo,
|
|
573
|
+
whole: Uint8Array | undefined,
|
|
574
|
+
read: (offset?: number, length?: number) => ReadableStream<Uint8Array>,
|
|
575
|
+
) {
|
|
576
|
+
super(info.size, () => (whole === undefined ? read() : streamOf(whole)))
|
|
577
|
+
this.path = info.path
|
|
578
|
+
this.etag = info.etag
|
|
579
|
+
this.contentType = info.contentType
|
|
580
|
+
this.lastModified = info.lastModified
|
|
581
|
+
this.expires = info.expires
|
|
582
|
+
this.meta = info.meta
|
|
583
|
+
this.#whole = whole
|
|
584
|
+
this.#read = read
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
/**
|
|
588
|
+
* A range of its bytes, as Blob.slice takes one: an end left out is the
|
|
589
|
+
* file's, and a negative place counts back from it, so `slice(-500)` is the
|
|
590
|
+
* last 500 bytes.
|
|
591
|
+
*/
|
|
592
|
+
slice(start?: number, end?: number): FileBytes {
|
|
593
|
+
const from = placeOf(start, this.size, 0)
|
|
594
|
+
const to = Math.max(from, placeOf(end, this.size, this.size))
|
|
595
|
+
const whole = this.#whole
|
|
596
|
+
if (whole !== undefined) {
|
|
597
|
+
const bytes = whole.subarray(from, to)
|
|
598
|
+
return new FileBytes(bytes.length, () => streamOf(bytes))
|
|
599
|
+
}
|
|
600
|
+
return new FileBytes(to - from, () => this.#read(from, to - from))
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
/** A place in bytes as Blob.slice reads one: negative from the end, and within the size. */
|
|
605
|
+
function placeOf(place: number | undefined, size: number, absent: number): number {
|
|
606
|
+
if (place === undefined) {
|
|
607
|
+
return absent
|
|
608
|
+
}
|
|
609
|
+
const at = Math.trunc(place)
|
|
610
|
+
return at < 0 ? Math.max(size + at, 0) : Math.min(at, size)
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
/** Bytes in hand as a stream, a copy, so that a reader's changes stay its own. */
|
|
614
|
+
function streamOf(bytes: Uint8Array): ReadableStream<Uint8Array> {
|
|
615
|
+
return new ReadableStream<Uint8Array>({
|
|
616
|
+
start(controller) {
|
|
617
|
+
if (bytes.length > 0) {
|
|
618
|
+
controller.enqueue(bytes.slice())
|
|
619
|
+
}
|
|
620
|
+
controller.close()
|
|
621
|
+
},
|
|
622
|
+
})
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
/** Enqueues a piece's bytes, none when it carries none. */
|
|
626
|
+
function enqueue(controller: ReadableStreamDefaultController<Uint8Array>, body: Uint8Array): void {
|
|
627
|
+
const bytes = BlobsPiece.decode(body).bytes ?? nothing
|
|
628
|
+
if (bytes.length > 0) {
|
|
629
|
+
controller.enqueue(bytes)
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
/**
|
|
634
|
+
* A body as its bytes come, its size and type when it says them, and whether
|
|
635
|
+
* it reads again from its start for a second try.
|
|
302
636
|
*/
|
|
303
637
|
interface Source {
|
|
304
638
|
size: number | undefined
|
|
305
639
|
contentType: string | undefined
|
|
306
640
|
replayable: boolean
|
|
307
|
-
chunks(): AsyncIterable<Uint8Array>
|
|
641
|
+
chunks(): Iterable<Uint8Array> | AsyncIterable<Uint8Array>
|
|
642
|
+
/** its bytes when they are known to fit a piece, for a write in one call */
|
|
643
|
+
whole(piece: number): Promise<Uint8Array | undefined>
|
|
308
644
|
}
|
|
309
645
|
|
|
310
|
-
function sourceOf(body: Body
|
|
311
|
-
|
|
312
|
-
body
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
if (
|
|
318
|
-
const bytes = body
|
|
646
|
+
function sourceOf(body: Body): Source {
|
|
647
|
+
const bytes =
|
|
648
|
+
typeof body === 'string'
|
|
649
|
+
? encoder.encode(body)
|
|
650
|
+
: body instanceof ArrayBuffer
|
|
651
|
+
? new Uint8Array(body)
|
|
652
|
+
: body
|
|
653
|
+
if (bytes instanceof Uint8Array) {
|
|
319
654
|
return {
|
|
320
|
-
size:
|
|
655
|
+
size: bytes.length,
|
|
321
656
|
contentType: undefined,
|
|
322
657
|
replayable: true,
|
|
323
|
-
chunks: () =>
|
|
658
|
+
chunks: () => [bytes],
|
|
659
|
+
whole: async piece => (bytes.length <= piece ? bytes : undefined),
|
|
324
660
|
}
|
|
325
661
|
}
|
|
326
|
-
if (
|
|
327
|
-
const blob =
|
|
328
|
-
const contentType = blob.type === '' ? undefined : blob.type
|
|
662
|
+
if (bytes instanceof Blob) {
|
|
663
|
+
const blob = bytes
|
|
329
664
|
return {
|
|
330
|
-
size:
|
|
331
|
-
contentType,
|
|
665
|
+
size: blob.size,
|
|
666
|
+
contentType: blob.type === '' ? undefined : blob.type,
|
|
332
667
|
replayable: true,
|
|
333
|
-
chunks: () =>
|
|
668
|
+
chunks: () => blob.stream() as AsyncIterable<Uint8Array>,
|
|
669
|
+
whole: async piece =>
|
|
670
|
+
blob.size <= piece ? new Uint8Array(await blob.arrayBuffer()) : undefined,
|
|
334
671
|
}
|
|
335
672
|
}
|
|
336
|
-
const stream =
|
|
337
|
-
return {
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
673
|
+
const stream = bytes as AsyncIterable<Uint8Array>
|
|
674
|
+
return {
|
|
675
|
+
size: undefined,
|
|
676
|
+
contentType: undefined,
|
|
677
|
+
replayable: false,
|
|
678
|
+
chunks: () => stream,
|
|
679
|
+
whole: async () => undefined,
|
|
343
680
|
}
|
|
344
681
|
}
|
|
345
682
|
|
|
346
|
-
|
|
347
|
-
|
|
683
|
+
/**
|
|
684
|
+
* The bytes of chunks again in pieces of `size` bytes, the last shorter. A
|
|
685
|
+
* chunk that holds a whole piece goes as it is, without a copy.
|
|
686
|
+
*/
|
|
687
|
+
async function* pieces(
|
|
688
|
+
chunks: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,
|
|
689
|
+
size: number,
|
|
348
690
|
): AsyncGenerator<Uint8Array> {
|
|
349
|
-
|
|
350
|
-
|
|
691
|
+
let piece = new Uint8Array(size)
|
|
692
|
+
let filled = 0
|
|
693
|
+
for await (const chunk of chunks) {
|
|
694
|
+
let at = 0
|
|
695
|
+
while (at < chunk.length) {
|
|
696
|
+
if (filled === 0 && chunk.length - at >= size) {
|
|
697
|
+
yield chunk.subarray(at, at + size)
|
|
698
|
+
at += size
|
|
699
|
+
continue
|
|
700
|
+
}
|
|
701
|
+
const taken = Math.min(size - filled, chunk.length - at)
|
|
702
|
+
piece.set(chunk.subarray(at, at + taken), filled)
|
|
703
|
+
filled += taken
|
|
704
|
+
at += taken
|
|
705
|
+
if (filled === size) {
|
|
706
|
+
yield piece
|
|
707
|
+
piece = new Uint8Array(size)
|
|
708
|
+
filled = 0
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
if (filled > 0) {
|
|
713
|
+
yield piece.subarray(0, filled)
|
|
351
714
|
}
|
|
352
715
|
}
|
|
353
716
|
|
|
354
717
|
/**
|
|
355
|
-
* Sends
|
|
356
|
-
* waits for
|
|
357
|
-
* DATA
|
|
718
|
+
* Sends an upload's pieces as DATA within the stream's credit, the last
|
|
719
|
+
* ending them, and waits for what the commit wrote. A connection lost before
|
|
720
|
+
* that last DATA left aborted the upload, which left nothing, and a body that
|
|
721
|
+
* reads again goes again on the next connection; lost after it, the
|
|
358
722
|
* commit's outcome is unknown.
|
|
359
723
|
*/
|
|
360
|
-
async function
|
|
361
|
-
|
|
362
|
-
request: Uint8Array,
|
|
724
|
+
async function uploaded(
|
|
725
|
+
stream: Stream,
|
|
363
726
|
source: Source,
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
727
|
+
piece: number,
|
|
728
|
+
signal: AbortSignal | undefined,
|
|
729
|
+
): Promise<FileInfo | undefined> {
|
|
367
730
|
const unwatch = watch(signal, stream)
|
|
368
|
-
let
|
|
731
|
+
let ending = false
|
|
369
732
|
try {
|
|
370
733
|
let last: Uint8Array | undefined
|
|
371
|
-
for await (const
|
|
372
|
-
if (piece.length === 0) {
|
|
373
|
-
continue
|
|
374
|
-
}
|
|
734
|
+
for await (const bytes of pieces(source.chunks(), piece)) {
|
|
375
735
|
if (last !== undefined) {
|
|
376
|
-
await stream.send(last, false)
|
|
736
|
+
await stream.send(BlobsPiece.encode({ bytes: last }), false)
|
|
377
737
|
}
|
|
378
|
-
last =
|
|
738
|
+
last = bytes
|
|
379
739
|
}
|
|
380
|
-
|
|
381
|
-
await stream.send(last ??
|
|
382
|
-
|
|
383
|
-
|
|
740
|
+
ending = true
|
|
741
|
+
await stream.send(BlobsPiece.encode({ bytes: last ?? nothing }), true)
|
|
742
|
+
// the RESPONSE that began it, and then what the commit wrote
|
|
743
|
+
await stream.next()
|
|
744
|
+
return writtenOf((await stream.next()).body)
|
|
384
745
|
} catch (err) {
|
|
385
|
-
if (err instanceof
|
|
386
|
-
if (ended) {
|
|
387
|
-
throw new OutcomeUnknownError(
|
|
388
|
-
`the connection was lost as the upload committed: ${err.message}`,
|
|
389
|
-
)
|
|
390
|
-
}
|
|
391
|
-
if (!source.replayable) {
|
|
392
|
-
throw new ClosedError(
|
|
393
|
-
`the connection was lost during the upload, which left nothing: ${err.message}`,
|
|
394
|
-
)
|
|
395
|
-
}
|
|
746
|
+
if (err instanceof ConnectionLostError && !ending && source.replayable) {
|
|
396
747
|
// the server aborted what it had of it, so the whole body goes again
|
|
397
|
-
throw new
|
|
748
|
+
throw new ConnectionLostError(err.message, false)
|
|
398
749
|
}
|
|
399
750
|
stream.cancel()
|
|
400
|
-
throw err
|
|
751
|
+
throw lostAs(err, ending)
|
|
401
752
|
} finally {
|
|
402
753
|
unwatch()
|
|
403
754
|
}
|
|
404
755
|
}
|
|
405
756
|
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
},
|
|
428
|
-
})
|
|
429
|
-
const bytes = async () => new Uint8Array(await new Response(body).arrayBuffer())
|
|
757
|
+
/** What a lost connection means for an upload: nothing left before its last DATA, unknown after. */
|
|
758
|
+
function lostAs(err: unknown, ending: boolean): unknown {
|
|
759
|
+
if (!(err instanceof ConnectionLostError)) {
|
|
760
|
+
return err
|
|
761
|
+
}
|
|
762
|
+
if (ending) {
|
|
763
|
+
return new OutcomeUnknownError(
|
|
764
|
+
`the connection was lost as the upload committed: ${err.message}`,
|
|
765
|
+
)
|
|
766
|
+
}
|
|
767
|
+
return new ClosedError(
|
|
768
|
+
`the connection was lost during the upload, which left nothing: ${err.message}`,
|
|
769
|
+
)
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
function writtenOf(body: Uint8Array): FileInfo | undefined {
|
|
773
|
+
const written = BlobsWritten.decode(body)
|
|
774
|
+
return written.info === undefined ? undefined : infoOf(written.info)
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
function infoOf(info: ReturnType<typeof BlobsInfo.decode>): FileInfo {
|
|
430
778
|
return {
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
779
|
+
path: info.path ?? '',
|
|
780
|
+
size: info.size ?? 0,
|
|
781
|
+
etag: info.etag ?? '',
|
|
782
|
+
contentType: info.contentType ?? '',
|
|
783
|
+
lastModified: new Date(info.lastModified ?? 0),
|
|
784
|
+
expires: dateOf(info.expires),
|
|
785
|
+
meta: info.meta ?? {},
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
/** A folder's segment as text, an integer its decimal spelling; the server refuses one that is not a segment. */
|
|
790
|
+
function segmentOf(segment: string | number | bigint): string {
|
|
791
|
+
if (typeof segment === 'number' && !Number.isSafeInteger(segment)) {
|
|
792
|
+
throw new InvalidError(`the folder segment ${segment}: a segment is text or an integer`)
|
|
793
|
+
}
|
|
794
|
+
return String(segment)
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
const binary: Record<string, number> = { KiB: 2 ** 10, MiB: 2 ** 20, GiB: 2 ** 30, TiB: 2 ** 40 }
|
|
798
|
+
|
|
799
|
+
/** A size's bytes. '2GB' is refused: it is 10⁹ bytes to some and 2³⁰ to others. */
|
|
800
|
+
function bytesOf(size: Size | string): number {
|
|
801
|
+
if (typeof size === 'number') {
|
|
802
|
+
if (!Number.isSafeInteger(size) || size < 0) {
|
|
803
|
+
throw new InvalidError(`a size of ${size} bytes`)
|
|
804
|
+
}
|
|
805
|
+
return size
|
|
806
|
+
}
|
|
807
|
+
const parts = /^(\d+(?:\.\d+)?)(KiB|MiB|GiB|TiB)$/.exec(size.trim())
|
|
808
|
+
if (parts === null) {
|
|
809
|
+
throw new InvalidError(
|
|
810
|
+
`the size ${JSON.stringify(size)}; write it in bytes, or as 64KiB, 10MiB, 2GiB or 1TiB`,
|
|
811
|
+
)
|
|
436
812
|
}
|
|
813
|
+
return Math.floor(Number(parts[1]) * binary[parts[2]!]!)
|
|
437
814
|
}
|