@wherever-dev/client 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/dist/client.d.ts +94 -2
- package/dist/client.js +980 -63
- package/dist/client.js.map +1 -1
- package/dist/types.d.ts +68 -0
- package/dist/types.js.map +1 -1
- package/package.json +1 -1
- package/src/client.ts +1051 -63
- package/src/types.ts +94 -0
- package/test/chunked-upload.test.ts +483 -0
- package/test/harness.ts +38 -2
- package/test/hydration-seam.test.ts +400 -0
- package/test/liveness-probe.test.ts +183 -0
- package/test/session-reload.test.ts +82 -0
- package/test/upload-survives-reconnect.test.ts +482 -0
package/src/types.ts
CHANGED
|
@@ -264,6 +264,18 @@ export interface WhereverState {
|
|
|
264
264
|
historyTotalCount: number;
|
|
265
265
|
historyOffset: number;
|
|
266
266
|
loadingMoreHistory: boolean;
|
|
267
|
+
// True once the server's `message_history` for the active session has landed.
|
|
268
|
+
// False at hydration (the messages came from the on-device cache) and on every
|
|
269
|
+
// reconnect rejoin, so `historyTotalCount` / `historyOffset` may be stale:
|
|
270
|
+
// `loadMoreHistory()` is a no-op while false (ADR 0013).
|
|
271
|
+
historyAuthoritative: boolean;
|
|
272
|
+
// The STAMPED identity (`serverKey(config)`) of the server that delivered the
|
|
273
|
+
// active session: set on `session_created` from the key captured when that
|
|
274
|
+
// socket was opened, and at hydration from the cached record. Never
|
|
275
|
+
// re-derived from the current settings (they can be edited without a
|
|
276
|
+
// reconnect). The web writes its conversation cache under exactly this key.
|
|
277
|
+
// null when there is no stamped session.
|
|
278
|
+
serverKey: string | null;
|
|
267
279
|
// Context-window usage for the active session (drives the "11.3% / 1.0M"
|
|
268
280
|
// indicator). null when unknown (no model, no usage yet, or just compacted).
|
|
269
281
|
contextUsage: ContextUsage | null;
|
|
@@ -283,6 +295,34 @@ export interface WhereverState {
|
|
|
283
295
|
skills: SkillCommand[];
|
|
284
296
|
}
|
|
285
297
|
|
|
298
|
+
/**
|
|
299
|
+
* A conversation read back from the web's on-device cache, handed to
|
|
300
|
+
* `WhereverClient.hydrateCachedSession()` so it can be shown before the socket
|
|
301
|
+
* is up (ADR 0013). Structurally a subset of the web cache's record, so a record
|
|
302
|
+
* can be passed as-is.
|
|
303
|
+
*/
|
|
304
|
+
export interface HydrationSnapshot {
|
|
305
|
+
/** The stamp the record was written under. Defaults to `serverKey(config)`. */
|
|
306
|
+
serverKey?: string;
|
|
307
|
+
sessionId: string;
|
|
308
|
+
sessionFile: string;
|
|
309
|
+
cwd: string | null;
|
|
310
|
+
model: string | null;
|
|
311
|
+
messages: ChatMessage[];
|
|
312
|
+
historyTotalCount?: number;
|
|
313
|
+
historyOffset?: number;
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
/**
|
|
317
|
+
* Synthesized by the client (never sent by the server) and delivered to
|
|
318
|
+
* `onMessage` listeners when the server refused the load of a HYDRATED session
|
|
319
|
+
* outright: the cached record is dead and should be evicted.
|
|
320
|
+
*/
|
|
321
|
+
export interface HydrationRejectedMessage {
|
|
322
|
+
type: 'hydration_rejected';
|
|
323
|
+
sessionId: string;
|
|
324
|
+
}
|
|
325
|
+
|
|
286
326
|
export interface SkillCommand {
|
|
287
327
|
name: string;
|
|
288
328
|
description?: string;
|
|
@@ -350,3 +390,57 @@ export function skillInvocationIdentity(content: string): string | null {
|
|
|
350
390
|
if (!parsed) return null;
|
|
351
391
|
return `skill:${parsed.skillName}\u0000${parsed.args}`;
|
|
352
392
|
}
|
|
393
|
+
|
|
394
|
+
// --- Uploads ---------------------------------------------------------------
|
|
395
|
+
|
|
396
|
+
/**
|
|
397
|
+
* The bytes of a file to upload, without any platform type (`File` / `Blob`):
|
|
398
|
+
* the client package is platform-neutral, so the web implements this over
|
|
399
|
+
* `File.slice().arrayBuffer()`. `read` may throw a `NotReadableError` (iOS and
|
|
400
|
+
* Android can invalidate a picked file once the page was backgrounded); the
|
|
401
|
+
* upload then fails with code `not_readable` ("pick it again").
|
|
402
|
+
*/
|
|
403
|
+
export interface UploadByteSource {
|
|
404
|
+
size: number;
|
|
405
|
+
read(offset: number, length: number): Promise<Uint8Array>;
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/**
|
|
409
|
+
* Lifecycle of one upload. `queued` (not sent yet: no socket, held for a rejoin,
|
|
410
|
+
* or the socket's capabilities are not known yet) -> `sending` -> `paused`
|
|
411
|
+
* (its socket was lost; it waits for the next one) -> `finishing` (chunked only:
|
|
412
|
+
* every byte acked, `upload_finish` sent) -> `done` | `failed` | `aborted`.
|
|
413
|
+
*/
|
|
414
|
+
export type UploadState =
|
|
415
|
+
| 'queued'
|
|
416
|
+
| 'sending'
|
|
417
|
+
| 'paused'
|
|
418
|
+
| 'finishing'
|
|
419
|
+
| 'done'
|
|
420
|
+
| 'failed'
|
|
421
|
+
| 'aborted';
|
|
422
|
+
|
|
423
|
+
/** `code` lets a UI react without matching on text. */
|
|
424
|
+
export interface UploadError {
|
|
425
|
+
message: string;
|
|
426
|
+
code?: 'not_readable';
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
/**
|
|
430
|
+
* Progress of one upload: the store entry (`WhereverClient.uploadsStore`, keyed
|
|
431
|
+
* by uploadId) and the `onProgress` payload share this shape. `acked` is the
|
|
432
|
+
* byte count the SERVER acknowledged (never what was merely sent); the legacy
|
|
433
|
+
* one-frame protocol reports 0 until done.
|
|
434
|
+
*/
|
|
435
|
+
export interface UploadProgress {
|
|
436
|
+
acked: number;
|
|
437
|
+
size: number;
|
|
438
|
+
state: UploadState;
|
|
439
|
+
error?: UploadError;
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
export interface UploadOptions {
|
|
443
|
+
signal?: AbortSignal;
|
|
444
|
+
mime?: string;
|
|
445
|
+
onProgress?: (progress: UploadProgress) => void;
|
|
446
|
+
}
|
|
@@ -0,0 +1,483 @@
|
|
|
1
|
+
import {describe, it, expect, beforeEach, afterEach, vi} from 'vitest';
|
|
2
|
+
import {get} from 'sveltore';
|
|
3
|
+
import {WhereverClient} from '../src/client.js';
|
|
4
|
+
import type {UploadProgress} from '../src/types.js';
|
|
5
|
+
import {makeWSFactory, type FakeWebSocket} from './harness.js';
|
|
6
|
+
|
|
7
|
+
// Resumable chunked uploads (upload.v1, docs/adr/0012): the file goes out as
|
|
8
|
+
// binary chunks, progress comes from the server's acks, a dropped socket pauses
|
|
9
|
+
// the upload, and the rejoin resumes it from the server's offset.
|
|
10
|
+
|
|
11
|
+
const KiB = 1024;
|
|
12
|
+
const MiB = 1024 * KiB;
|
|
13
|
+
|
|
14
|
+
function makeSource(size: number) {
|
|
15
|
+
const data = new Uint8Array(size);
|
|
16
|
+
for (let i = 0; i < size; i++) data[i] = i % 251;
|
|
17
|
+
const reads: Array<[number, number]> = [];
|
|
18
|
+
return {
|
|
19
|
+
data,
|
|
20
|
+
reads,
|
|
21
|
+
source: {
|
|
22
|
+
size,
|
|
23
|
+
read: async (offset: number, length: number) => {
|
|
24
|
+
reads.push([offset, length]);
|
|
25
|
+
return data.slice(offset, offset + length);
|
|
26
|
+
},
|
|
27
|
+
},
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Byte-for-byte equality without vitest's slow per-element deep diff.
|
|
32
|
+
function sameBytes(a: Uint8Array, b: Uint8Array) {
|
|
33
|
+
return Buffer.from(a.buffer, a.byteOffset, a.byteLength).equals(Buffer.from(b.buffer, b.byteOffset, b.byteLength));
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// Let the async byte-source reads (and the sends after them) run.
|
|
37
|
+
async function flush() {
|
|
38
|
+
for (let i = 0; i < 50; i++) await Promise.resolve();
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const SESSION = {
|
|
42
|
+
type: 'session_created',
|
|
43
|
+
sessionId: 'sid-1',
|
|
44
|
+
sessionFile: '/tmp/project/session.jsonl',
|
|
45
|
+
cwd: '/tmp/project',
|
|
46
|
+
model: 'fake:model',
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
describe('chunked uploads (upload.v1)', () => {
|
|
50
|
+
let ws: ReturnType<typeof makeWSFactory>;
|
|
51
|
+
let client: WhereverClient;
|
|
52
|
+
|
|
53
|
+
beforeEach(() => {
|
|
54
|
+
vi.useFakeTimers();
|
|
55
|
+
ws = makeWSFactory();
|
|
56
|
+
client = new WhereverClient({
|
|
57
|
+
host: 'localhost',
|
|
58
|
+
port: 1234,
|
|
59
|
+
secure: false,
|
|
60
|
+
WebSocketCtor: ws.ctor,
|
|
61
|
+
});
|
|
62
|
+
client.connect();
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
afterEach(() => {
|
|
66
|
+
client.disconnect(true);
|
|
67
|
+
vi.useRealTimers();
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
function connected(sock: FakeWebSocket, capabilities?: string[]) {
|
|
71
|
+
sock.receive({type: 'connected', clientId: 'c1', serverVersion: 'x', capabilities});
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function openAndJoin() {
|
|
75
|
+
const sock = ws.last();
|
|
76
|
+
sock.open();
|
|
77
|
+
connected(sock, ['upload.v1']);
|
|
78
|
+
client.joinSession('/tmp/project/session.jsonl');
|
|
79
|
+
sock.receive(SESSION);
|
|
80
|
+
sock.receive({type: 'message_history', sessionId: 'sid-1', messages: [], totalCount: 0, offset: 0});
|
|
81
|
+
return sock;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// `null`: an older server, whose `connected` carries no capabilities.
|
|
85
|
+
function dropAndReconnect(capabilities: string[] | null = ['upload.v1']) {
|
|
86
|
+
const first = ws.last();
|
|
87
|
+
first.close();
|
|
88
|
+
vi.advanceTimersByTime(5000);
|
|
89
|
+
const second = ws.last();
|
|
90
|
+
expect(second).not.toBe(first);
|
|
91
|
+
second.open();
|
|
92
|
+
connected(second, capabilities ?? undefined);
|
|
93
|
+
return second;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// Ack every chunk this socket has sent beyond `fromIndex`, as a server would.
|
|
97
|
+
async function ackChunks(sock: FakeWebSocket, fromIndex: number) {
|
|
98
|
+
let i = fromIndex;
|
|
99
|
+
while (i < sock.binary.length) {
|
|
100
|
+
const c = sock.binary[i++];
|
|
101
|
+
sock.receive({type: 'upload_offset', uploadId: c.uploadId, offset: c.offset + c.payload.length});
|
|
102
|
+
await flush();
|
|
103
|
+
}
|
|
104
|
+
return i;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
async function ackUntilFinish(sock: FakeWebSocket, fromIndex = 0) {
|
|
108
|
+
let i = fromIndex;
|
|
109
|
+
for (let guard = 0; guard < 1000 && !sock.lastSentOfType('upload_finish'); guard++) {
|
|
110
|
+
i = await ackChunks(sock, i);
|
|
111
|
+
if (i >= sock.binary.length && !sock.lastSentOfType('upload_finish')) await flush();
|
|
112
|
+
}
|
|
113
|
+
return i;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function bytesOf(sock: FakeWebSocket) {
|
|
117
|
+
return sock.binary.map((c) => [c.offset, c.payload.length]);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
it('uploads in binary chunks; progress reflects acked bytes only', async () => {
|
|
121
|
+
const sock = openAndJoin();
|
|
122
|
+
const {data, source} = makeSource(600 * KiB);
|
|
123
|
+
const progress: UploadProgress[] = [];
|
|
124
|
+
const p = client.uploadFileViaWebSocket('sid-1', 'clip.mp4', source, {
|
|
125
|
+
mime: 'video/mp4',
|
|
126
|
+
onProgress: (pr) => progress.push(pr),
|
|
127
|
+
});
|
|
128
|
+
const begin = sock.lastSentOfType('upload_begin');
|
|
129
|
+
expect(begin).toMatchObject({sessionId: 'sid-1', filename: 'clip.mp4', size: 600 * KiB, mime: 'video/mp4'});
|
|
130
|
+
expect(begin.uploadId).toMatch(/^[A-Za-z0-9_-]{22,64}$/);
|
|
131
|
+
expect(sock.lastSentOfType('file_upload')).toBeUndefined();
|
|
132
|
+
expect(progress.at(-1)).toMatchObject({state: 'sending', acked: 0, size: 600 * KiB});
|
|
133
|
+
// Nothing goes out before the server says where to start.
|
|
134
|
+
await flush();
|
|
135
|
+
expect(sock.binary).toHaveLength(0);
|
|
136
|
+
|
|
137
|
+
sock.receive({type: 'upload_offset', uploadId: begin.uploadId, offset: 0});
|
|
138
|
+
await flush();
|
|
139
|
+
expect(sock.binary.length).toBeGreaterThan(0);
|
|
140
|
+
const first = sock.binary[0];
|
|
141
|
+
expect(first).toMatchObject({kind: 1, version: 1, uploadId: begin.uploadId, offset: 0});
|
|
142
|
+
// Sent but not acked: progress has not moved.
|
|
143
|
+
expect(progress.at(-1)).toMatchObject({acked: 0});
|
|
144
|
+
expect(get(client.uploadsStore)[begin.uploadId]).toMatchObject({acked: 0, state: 'sending'});
|
|
145
|
+
|
|
146
|
+
sock.receive({type: 'upload_offset', uploadId: begin.uploadId, offset: first.payload.length});
|
|
147
|
+
await flush();
|
|
148
|
+
expect(progress.at(-1)).toMatchObject({acked: first.payload.length, state: 'sending'});
|
|
149
|
+
|
|
150
|
+
await ackUntilFinish(sock, 1);
|
|
151
|
+
const finish = sock.lastSentOfType('upload_finish');
|
|
152
|
+
expect(finish).toMatchObject({uploadId: begin.uploadId, sessionId: 'sid-1'});
|
|
153
|
+
expect(progress.at(-1)).toMatchObject({acked: 600 * KiB, state: 'finishing'});
|
|
154
|
+
|
|
155
|
+
// The bytes on the wire are exactly the file, in order, each byte once.
|
|
156
|
+
const joined = new Uint8Array(600 * KiB);
|
|
157
|
+
let at = 0;
|
|
158
|
+
for (const c of sock.binary) {
|
|
159
|
+
expect(c.offset).toBe(at);
|
|
160
|
+
joined.set(c.payload, c.offset);
|
|
161
|
+
at += c.payload.length;
|
|
162
|
+
}
|
|
163
|
+
expect(at).toBe(600 * KiB);
|
|
164
|
+
expect(sameBytes(joined, data)).toBe(true);
|
|
165
|
+
// Every acked value reported was one the server sent (monotonic).
|
|
166
|
+
const acks = progress.map((pr) => pr.acked);
|
|
167
|
+
expect([...acks].sort((a, b) => a - b)).toEqual(acks);
|
|
168
|
+
|
|
169
|
+
sock.receive({type: 'file_uploaded', uploadId: begin.uploadId, sessionId: 'sid-1', filename: 'clip.mp4', savedPath: '/up/1_clip.mp4'});
|
|
170
|
+
await expect(p).resolves.toEqual({savedPath: '/up/1_clip.mp4', filename: 'clip.mp4'});
|
|
171
|
+
expect(progress.at(-1)).toMatchObject({acked: 600 * KiB, state: 'done'});
|
|
172
|
+
expect(get(client.uploadsStore)[begin.uploadId]).toMatchObject({state: 'done'});
|
|
173
|
+
expect(client.pendingUploadCount()).toBe(0);
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
it('paces by bytes in flight: 64 KiB at first, at most 256 KiB chunks and 1 MiB in flight', async () => {
|
|
177
|
+
const sock = openAndJoin();
|
|
178
|
+
const {source} = makeSource(4 * MiB);
|
|
179
|
+
void client.uploadFileViaWebSocket('sid-1', 'big.bin', source).catch(() => {});
|
|
180
|
+
const {uploadId} = sock.lastSentOfType('upload_begin');
|
|
181
|
+
sock.receive({type: 'upload_offset', uploadId, offset: 0});
|
|
182
|
+
await flush();
|
|
183
|
+
// No throughput measured yet: the minimum window, one 64 KiB chunk.
|
|
184
|
+
expect(bytesOf(sock)).toEqual([[0, 64 * KiB]]);
|
|
185
|
+
|
|
186
|
+
// A fast ack (no time elapsed): the window opens to its 1 MiB maximum,
|
|
187
|
+
// filled with 256 KiB chunks.
|
|
188
|
+
sock.receive({type: 'upload_offset', uploadId, offset: 64 * KiB});
|
|
189
|
+
await flush();
|
|
190
|
+
const after = sock.binary.slice(1);
|
|
191
|
+
expect(after.length).toBeGreaterThan(1);
|
|
192
|
+
for (const c of after) expect(c.payload.length).toBeLessThanOrEqual(256 * KiB);
|
|
193
|
+
const inFlight = after.reduce((n, c) => n + c.payload.length, 0);
|
|
194
|
+
expect(inFlight).toBeLessThanOrEqual(1 * MiB);
|
|
195
|
+
expect(inFlight).toBeGreaterThan(512 * KiB);
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
it('a slow link keeps the window at its 64 KiB floor', async () => {
|
|
199
|
+
const sock = openAndJoin();
|
|
200
|
+
const {source} = makeSource(1 * MiB);
|
|
201
|
+
void client.uploadFileViaWebSocket('sid-1', 'slow.bin', source).catch(() => {});
|
|
202
|
+
const {uploadId} = sock.lastSentOfType('upload_begin');
|
|
203
|
+
sock.receive({type: 'upload_offset', uploadId, offset: 0});
|
|
204
|
+
await flush();
|
|
205
|
+
for (let i = 0; i < 4; i++) {
|
|
206
|
+
// 64 KiB every 4 s: 16 KiB/s, so 2 s worth is far below the floor.
|
|
207
|
+
// Pongs keep the liveness watchdog satisfied meanwhile.
|
|
208
|
+
sock.receive({type: 'pong', timestamp: i});
|
|
209
|
+
vi.advanceTimersByTime(4000);
|
|
210
|
+
const c = sock.binary.at(-1)!;
|
|
211
|
+
sock.receive({type: 'upload_offset', uploadId, offset: c.offset + c.payload.length});
|
|
212
|
+
await flush();
|
|
213
|
+
}
|
|
214
|
+
for (const c of sock.binary) expect(c.payload.length).toBe(64 * KiB);
|
|
215
|
+
expect(sock.binary).toHaveLength(5);
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
it('a drop mid-upload pauses it; the rejoin resumes from the server offset without re-sending acked bytes', async () => {
|
|
219
|
+
openAndJoin();
|
|
220
|
+
const {data, source} = makeSource(1 * MiB);
|
|
221
|
+
const progress: UploadProgress[] = [];
|
|
222
|
+
const p = client.uploadFileViaWebSocket('sid-1', 'r.bin', source, {onProgress: (pr) => progress.push(pr)});
|
|
223
|
+
const first = ws.last();
|
|
224
|
+
const {uploadId} = first.lastSentOfType('upload_begin');
|
|
225
|
+
first.receive({type: 'upload_offset', uploadId, offset: 0});
|
|
226
|
+
await flush();
|
|
227
|
+
// Ack the first chunk only, then lose the socket with more in flight.
|
|
228
|
+
const c0 = first.binary[0];
|
|
229
|
+
first.receive({type: 'upload_offset', uploadId, offset: c0.payload.length});
|
|
230
|
+
await flush();
|
|
231
|
+
const acked = c0.payload.length;
|
|
232
|
+
|
|
233
|
+
const second = dropAndReconnect();
|
|
234
|
+
expect(progress.at(-1)).toMatchObject({state: 'paused', acked});
|
|
235
|
+
// Held until the rejoin is answered: session_load first, no begin yet.
|
|
236
|
+
expect(second.lastSentOfType('session_load')).toBeTruthy();
|
|
237
|
+
expect(second.lastSentOfType('upload_begin')).toBeUndefined();
|
|
238
|
+
|
|
239
|
+
second.receive(SESSION);
|
|
240
|
+
const begin = second.lastSentOfType('upload_begin');
|
|
241
|
+
expect(begin).toMatchObject({uploadId, sessionId: 'sid-1', size: 1 * MiB, filename: 'r.bin'});
|
|
242
|
+
expect(second.binary).toHaveLength(0);
|
|
243
|
+
|
|
244
|
+
// The server kept exactly the acked bytes.
|
|
245
|
+
second.receive({type: 'upload_offset', uploadId, offset: acked});
|
|
246
|
+
await flush();
|
|
247
|
+
expect(second.binary[0].offset).toBe(acked);
|
|
248
|
+
await ackUntilFinish(second);
|
|
249
|
+
for (const c of second.binary) expect(c.offset).toBeGreaterThanOrEqual(acked);
|
|
250
|
+
const joined = new Uint8Array(1 * MiB);
|
|
251
|
+
joined.set(data.subarray(0, acked));
|
|
252
|
+
for (const c of second.binary) joined.set(c.payload, c.offset);
|
|
253
|
+
expect(sameBytes(joined, data)).toBe(true);
|
|
254
|
+
|
|
255
|
+
second.receive({type: 'file_uploaded', uploadId, sessionId: 'sid-1', filename: 'r.bin', savedPath: '/up/r.bin'});
|
|
256
|
+
await expect(p).resolves.toMatchObject({savedPath: '/up/r.bin'});
|
|
257
|
+
});
|
|
258
|
+
|
|
259
|
+
it('an offset mismatch realigns: in-flight acks drain, then it continues from the true offset', async () => {
|
|
260
|
+
const sock = openAndJoin();
|
|
261
|
+
const {source} = makeSource(2 * MiB);
|
|
262
|
+
void client.uploadFileViaWebSocket('sid-1', 'm.bin', source).catch(() => {});
|
|
263
|
+
const {uploadId} = sock.lastSentOfType('upload_begin');
|
|
264
|
+
sock.receive({type: 'upload_offset', uploadId, offset: 0});
|
|
265
|
+
await flush();
|
|
266
|
+
sock.receive({type: 'upload_offset', uploadId, offset: 64 * KiB});
|
|
267
|
+
await flush();
|
|
268
|
+
const inFlight = sock.binary.slice(1);
|
|
269
|
+
expect(inFlight.length).toBeGreaterThan(1);
|
|
270
|
+
const sentBefore = sock.binary.length;
|
|
271
|
+
|
|
272
|
+
// The server's true offset is NOT where the first in-flight chunk began
|
|
273
|
+
// (e.g. it lost a write): it answers with its offset, 32 KiB.
|
|
274
|
+
sock.receive({type: 'upload_offset', uploadId, offset: 32 * KiB});
|
|
275
|
+
await flush();
|
|
276
|
+
// Nothing new until the other in-flight chunks are answered too.
|
|
277
|
+
expect(sock.binary).toHaveLength(sentBefore);
|
|
278
|
+
for (let i = 1; i < inFlight.length; i++) {
|
|
279
|
+
sock.receive({type: 'upload_offset', uploadId, offset: 32 * KiB});
|
|
280
|
+
await flush();
|
|
281
|
+
if (i < inFlight.length - 1) expect(sock.binary).toHaveLength(sentBefore);
|
|
282
|
+
}
|
|
283
|
+
expect(sock.binary.length).toBeGreaterThan(sentBefore);
|
|
284
|
+
expect(sock.binary[sentBefore].offset).toBe(32 * KiB);
|
|
285
|
+
expect(get(client.uploadsStore)[uploadId]).toMatchObject({acked: 32 * KiB});
|
|
286
|
+
});
|
|
287
|
+
|
|
288
|
+
it('done: true on the resume begin resolves without re-sending anything', async () => {
|
|
289
|
+
openAndJoin();
|
|
290
|
+
const {source} = makeSource(100 * KiB);
|
|
291
|
+
const p = client.uploadFileViaWebSocket('sid-1', 'd.png', source);
|
|
292
|
+
const first = ws.last();
|
|
293
|
+
const {uploadId} = first.lastSentOfType('upload_begin');
|
|
294
|
+
first.receive({type: 'upload_offset', uploadId, offset: 0});
|
|
295
|
+
await flush();
|
|
296
|
+
await ackUntilFinish(first);
|
|
297
|
+
expect(first.lastSentOfType('upload_finish')).toBeTruthy();
|
|
298
|
+
// file_uploaded is lost with the socket.
|
|
299
|
+
const second = dropAndReconnect();
|
|
300
|
+
second.receive(SESSION);
|
|
301
|
+
expect(second.lastSentOfType('upload_begin')).toMatchObject({uploadId});
|
|
302
|
+
second.receive({type: 'upload_offset', uploadId, offset: 100 * KiB, done: true, savedPath: '/up/d.png'});
|
|
303
|
+
await expect(p).resolves.toEqual({savedPath: '/up/d.png', filename: 'd.png'});
|
|
304
|
+
await flush();
|
|
305
|
+
expect(second.binary).toHaveLength(0);
|
|
306
|
+
expect(second.lastSentOfType('upload_finish')).toBeUndefined();
|
|
307
|
+
expect(get(client.uploadsStore)[uploadId]).toMatchObject({state: 'done', acked: 100 * KiB});
|
|
308
|
+
});
|
|
309
|
+
|
|
310
|
+
it('an empty file finishes right after the begin', async () => {
|
|
311
|
+
const sock = openAndJoin();
|
|
312
|
+
const {source} = makeSource(0);
|
|
313
|
+
const p = client.uploadFileViaWebSocket('sid-1', 'empty.txt', source);
|
|
314
|
+
const {uploadId} = sock.lastSentOfType('upload_begin');
|
|
315
|
+
sock.receive({type: 'upload_offset', uploadId, offset: 0});
|
|
316
|
+
await flush();
|
|
317
|
+
expect(sock.binary).toHaveLength(0);
|
|
318
|
+
expect(sock.lastSentOfType('upload_finish')).toMatchObject({uploadId});
|
|
319
|
+
sock.receive({type: 'file_uploaded', uploadId, sessionId: 'sid-1', filename: 'empty.txt', savedPath: '/up/e'});
|
|
320
|
+
await expect(p).resolves.toMatchObject({savedPath: '/up/e'});
|
|
321
|
+
});
|
|
322
|
+
|
|
323
|
+
it('cancel sends upload_abort for an in-flight chunked upload and stops sending', async () => {
|
|
324
|
+
const sock = openAndJoin();
|
|
325
|
+
const {source} = makeSource(1 * MiB);
|
|
326
|
+
const controller = new AbortController();
|
|
327
|
+
const progress: UploadProgress[] = [];
|
|
328
|
+
const p = client.uploadFileViaWebSocket('sid-1', 'c.bin', source, {
|
|
329
|
+
signal: controller.signal,
|
|
330
|
+
onProgress: (pr) => progress.push(pr),
|
|
331
|
+
});
|
|
332
|
+
const settled = expect(p).rejects.toThrow(/cancelled/);
|
|
333
|
+
const {uploadId} = sock.lastSentOfType('upload_begin');
|
|
334
|
+
sock.receive({type: 'upload_offset', uploadId, offset: 0});
|
|
335
|
+
await flush();
|
|
336
|
+
controller.abort();
|
|
337
|
+
await settled;
|
|
338
|
+
expect(sock.lastSentOfType('upload_abort')).toEqual({type: 'upload_abort', uploadId, sessionId: 'sid-1'});
|
|
339
|
+
expect(progress.at(-1)).toMatchObject({state: 'aborted'});
|
|
340
|
+
const sent = sock.binary.length;
|
|
341
|
+
sock.receive({type: 'upload_offset', uploadId, offset: 64 * KiB});
|
|
342
|
+
await flush();
|
|
343
|
+
expect(sock.binary).toHaveLength(sent);
|
|
344
|
+
expect(client.pendingUploadCount()).toBe(0);
|
|
345
|
+
});
|
|
346
|
+
|
|
347
|
+
it('a NotReadableError fails the upload with the "pick it again" message and code', async () => {
|
|
348
|
+
const sock = openAndJoin();
|
|
349
|
+
const progress: UploadProgress[] = [];
|
|
350
|
+
const source = {
|
|
351
|
+
size: 10 * KiB,
|
|
352
|
+
read: async () => {
|
|
353
|
+
const err = new Error('The requested file could not be read');
|
|
354
|
+
err.name = 'NotReadableError';
|
|
355
|
+
throw err;
|
|
356
|
+
},
|
|
357
|
+
};
|
|
358
|
+
const p = client.uploadFileViaWebSocket('sid-1', 'gone.jpg', source, {onProgress: (pr) => progress.push(pr)});
|
|
359
|
+
const settled = p.catch((e) => e);
|
|
360
|
+
const {uploadId} = sock.lastSentOfType('upload_begin');
|
|
361
|
+
sock.receive({type: 'upload_offset', uploadId, offset: 0});
|
|
362
|
+
const err = await settled;
|
|
363
|
+
expect(err.message).toMatch(/no longer readable, pick it again/);
|
|
364
|
+
expect(err.code).toBe('not_readable');
|
|
365
|
+
expect(progress.at(-1)).toMatchObject({
|
|
366
|
+
state: 'failed',
|
|
367
|
+
error: {message: expect.stringMatching(/pick it again/), code: 'not_readable'},
|
|
368
|
+
});
|
|
369
|
+
expect(get(client.uploadsStore)[uploadId].error?.code).toBe('not_readable');
|
|
370
|
+
});
|
|
371
|
+
|
|
372
|
+
it('a server error fails the chunked upload with its reason', async () => {
|
|
373
|
+
const sock = openAndJoin();
|
|
374
|
+
const {source} = makeSource(10 * KiB);
|
|
375
|
+
const p = client.uploadFileViaWebSocket('sid-1', 'x.bin', source);
|
|
376
|
+
const {uploadId} = sock.lastSentOfType('upload_begin');
|
|
377
|
+
sock.receive({type: 'file_upload_error', uploadId, sessionId: 'sid-1', error: 'read-only'});
|
|
378
|
+
await expect(p).rejects.toThrow('read-only');
|
|
379
|
+
expect(get(client.uploadsStore)[uploadId]).toMatchObject({state: 'failed', error: {message: 'read-only'}});
|
|
380
|
+
});
|
|
381
|
+
|
|
382
|
+
it('an upload is held while the socket capabilities are unknown, never sent as legacy', async () => {
|
|
383
|
+
const sock = ws.last();
|
|
384
|
+
sock.open();
|
|
385
|
+
const {source} = makeSource(10 * KiB);
|
|
386
|
+
void client.uploadFileViaWebSocket('sid-1', 'h.bin', source).catch(() => {});
|
|
387
|
+
await flush();
|
|
388
|
+
expect(sock.lastSentOfType('file_upload')).toBeUndefined();
|
|
389
|
+
expect(sock.lastSentOfType('upload_begin')).toBeUndefined();
|
|
390
|
+
connected(sock, ['upload.v1']);
|
|
391
|
+
expect(sock.lastSentOfType('upload_begin')).toMatchObject({filename: 'h.bin'});
|
|
392
|
+
});
|
|
393
|
+
|
|
394
|
+
it('a server without upload.v1 gets the legacy base64 frame', async () => {
|
|
395
|
+
const sock = ws.last();
|
|
396
|
+
sock.open();
|
|
397
|
+
connected(sock);
|
|
398
|
+
const {data, source} = makeSource(5 * KiB);
|
|
399
|
+
void client.uploadFileViaWebSocket('sid-1', 'l.bin', source).catch(() => {});
|
|
400
|
+
await flush();
|
|
401
|
+
expect(sock.lastSentOfType('upload_begin')).toBeUndefined();
|
|
402
|
+
const frame = sock.lastSentOfType('file_upload');
|
|
403
|
+
expect(frame).toMatchObject({sessionId: 'sid-1', filename: 'l.bin'});
|
|
404
|
+
expect(sameBytes(Buffer.from(frame.data, 'base64'), data)).toBe(true);
|
|
405
|
+
});
|
|
406
|
+
|
|
407
|
+
it('legacy base64 is encoded across reads with no padding mid-stream', async () => {
|
|
408
|
+
const sock = ws.last();
|
|
409
|
+
sock.open();
|
|
410
|
+
connected(sock);
|
|
411
|
+
// Larger than one encode chunk, and not a multiple of 3.
|
|
412
|
+
const {data, source, reads} = makeSource(2 * MiB + 1);
|
|
413
|
+
void client.uploadFileViaWebSocket('sid-1', 'l2.bin', source).catch(() => {});
|
|
414
|
+
await flush();
|
|
415
|
+
expect(reads.length).toBeGreaterThan(1);
|
|
416
|
+
for (const [, len] of reads.slice(0, -1)) expect(len % 3).toBe(0);
|
|
417
|
+
const frame = sock.lastSentOfType('file_upload');
|
|
418
|
+
expect(frame.data.slice(0, -4).includes('=')).toBe(false);
|
|
419
|
+
expect(sameBytes(Buffer.from(frame.data, 'base64'), data)).toBe(true);
|
|
420
|
+
});
|
|
421
|
+
|
|
422
|
+
it('a paused chunked upload that reconnects to a server without upload.v1 fails with the retry message', async () => {
|
|
423
|
+
openAndJoin();
|
|
424
|
+
const {source} = makeSource(1 * MiB);
|
|
425
|
+
const p = client.uploadFileViaWebSocket('sid-1', 'g.bin', source);
|
|
426
|
+
const first = ws.last();
|
|
427
|
+
const {uploadId} = first.lastSentOfType('upload_begin');
|
|
428
|
+
first.receive({type: 'upload_offset', uploadId, offset: 0});
|
|
429
|
+
await flush();
|
|
430
|
+
const settled = p.catch((e) => e);
|
|
431
|
+
const second = dropAndReconnect(null);
|
|
432
|
+
second.receive(SESSION);
|
|
433
|
+
const err = await settled;
|
|
434
|
+
expect(err.message).toMatch(/server no longer supports resumable uploads, retry/);
|
|
435
|
+
expect(second.lastSentOfType('file_upload')).toBeUndefined();
|
|
436
|
+
expect(second.lastSentOfType('upload_begin')).toBeUndefined();
|
|
437
|
+
expect(get(client.uploadsStore)[uploadId]).toMatchObject({state: 'failed'});
|
|
438
|
+
});
|
|
439
|
+
|
|
440
|
+
it('a paused chunked upload fails with "session changed" when the rejoin lands on another session', async () => {
|
|
441
|
+
openAndJoin();
|
|
442
|
+
const {source} = makeSource(1 * MiB);
|
|
443
|
+
const p = client.uploadFileViaWebSocket('sid-1', 's.bin', source);
|
|
444
|
+
const first = ws.last();
|
|
445
|
+
const {uploadId} = first.lastSentOfType('upload_begin');
|
|
446
|
+
first.receive({type: 'upload_offset', uploadId, offset: 0});
|
|
447
|
+
await flush();
|
|
448
|
+
const settled = expect(p).rejects.toThrow(/session changed/);
|
|
449
|
+
const second = dropAndReconnect();
|
|
450
|
+
second.receive({...SESSION, sessionId: 'sid-other'});
|
|
451
|
+
await settled;
|
|
452
|
+
expect(second.lastSentOfType('upload_begin')).toBeUndefined();
|
|
453
|
+
});
|
|
454
|
+
|
|
455
|
+
it('a paused chunked upload keeps the 10 min waiting clock', async () => {
|
|
456
|
+
openAndJoin();
|
|
457
|
+
const {source} = makeSource(1 * MiB);
|
|
458
|
+
const p = client.uploadFileViaWebSocket('sid-1', 't.bin', source);
|
|
459
|
+
const first = ws.last();
|
|
460
|
+
const {uploadId} = first.lastSentOfType('upload_begin');
|
|
461
|
+
first.receive({type: 'upload_offset', uploadId, offset: 0});
|
|
462
|
+
await flush();
|
|
463
|
+
const settled = expect(p).rejects.toThrow(/timed out/);
|
|
464
|
+
client.suspend();
|
|
465
|
+
vi.advanceTimersByTime(10 * 60_000 + 1);
|
|
466
|
+
await settled;
|
|
467
|
+
});
|
|
468
|
+
|
|
469
|
+
it('a chunked upload is not capped by the legacy send count', async () => {
|
|
470
|
+
openAndJoin();
|
|
471
|
+
const {source} = makeSource(1 * MiB);
|
|
472
|
+
const p = client.uploadFileViaWebSocket('sid-1', 'n.bin', source);
|
|
473
|
+
const {uploadId} = ws.last().lastSentOfType('upload_begin');
|
|
474
|
+
let sock = ws.last();
|
|
475
|
+
for (let i = 0; i < 4; i++) {
|
|
476
|
+
sock = dropAndReconnect();
|
|
477
|
+
sock.receive(SESSION);
|
|
478
|
+
expect(sock.lastSentOfType('upload_begin')).toMatchObject({uploadId});
|
|
479
|
+
}
|
|
480
|
+
sock.receive({type: 'upload_offset', uploadId, offset: 1 * MiB, done: true, savedPath: '/up/n'});
|
|
481
|
+
await expect(p).resolves.toMatchObject({savedPath: '/up/n'});
|
|
482
|
+
});
|
|
483
|
+
});
|
package/test/harness.ts
CHANGED
|
@@ -6,6 +6,31 @@
|
|
|
6
6
|
|
|
7
7
|
export type WSListener = (event: any) => void;
|
|
8
8
|
|
|
9
|
+
/** A decoded `upload.v1` chunk frame (see docs/USAGE.md). */
|
|
10
|
+
export interface BinaryChunk {
|
|
11
|
+
kind: number;
|
|
12
|
+
version: number;
|
|
13
|
+
uploadId: string;
|
|
14
|
+
offset: number;
|
|
15
|
+
payload: Uint8Array;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
// [u8 kind][u8 version][u8 idLen][uploadId ascii][u64 offset BE][payload]
|
|
19
|
+
export function decodeChunkFrame(bytes: Uint8Array): BinaryChunk {
|
|
20
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
21
|
+
const idLen = bytes[2];
|
|
22
|
+
const uploadId = String.fromCharCode(...bytes.subarray(3, 3 + idLen));
|
|
23
|
+
const at = 3 + idLen;
|
|
24
|
+
const offset = view.getUint32(at) * 2 ** 32 + view.getUint32(at + 4);
|
|
25
|
+
return {
|
|
26
|
+
kind: bytes[0],
|
|
27
|
+
version: bytes[1],
|
|
28
|
+
uploadId,
|
|
29
|
+
offset,
|
|
30
|
+
payload: bytes.slice(at + 8),
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
|
|
9
34
|
export class FakeWebSocket {
|
|
10
35
|
// WHATWG readyState constants.
|
|
11
36
|
static CONNECTING = 0;
|
|
@@ -16,6 +41,9 @@ export class FakeWebSocket {
|
|
|
16
41
|
url: string;
|
|
17
42
|
readyState = FakeWebSocket.CONNECTING;
|
|
18
43
|
sent: any[] = [];
|
|
44
|
+
// Binary frames (upload.v1 chunks), decoded, in send order. Kept apart from
|
|
45
|
+
// `sent` because they are not JSON.
|
|
46
|
+
binary: BinaryChunk[] = [];
|
|
19
47
|
private listeners: Record<string, Set<WSListener>> = {
|
|
20
48
|
open: new Set(),
|
|
21
49
|
message: new Set(),
|
|
@@ -41,8 +69,16 @@ export class FakeWebSocket {
|
|
|
41
69
|
}
|
|
42
70
|
}
|
|
43
71
|
|
|
44
|
-
send(data: string) {
|
|
45
|
-
|
|
72
|
+
send(data: string | ArrayBufferView | ArrayBuffer) {
|
|
73
|
+
if (typeof data === 'string') {
|
|
74
|
+
this.sent.push(JSON.parse(data));
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
const bytes =
|
|
78
|
+
data instanceof ArrayBuffer
|
|
79
|
+
? new Uint8Array(data)
|
|
80
|
+
: new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
81
|
+
this.binary.push(decodeChunkFrame(bytes));
|
|
46
82
|
}
|
|
47
83
|
|
|
48
84
|
close() {
|