@statewalker/webrun-files-rpc 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +264 -2
- package/dist/cjs/index.cjs +671 -0
- package/dist/client.d.ts +40 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/errors.d.ts +13 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/esm/index.js +666 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/keyed.d.ts +21 -0
- package/dist/keyed.d.ts.map +1 -0
- package/dist/server.d.ts +49 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/types.d.ts +80 -0
- package/dist/types.d.ts.map +1 -0
- package/package.json +54 -3
- package/src/client.ts +200 -0
- package/src/errors.ts +25 -0
- package/src/index.ts +18 -0
- package/src/keyed.ts +57 -0
- package/src/server.ts +591 -0
- package/src/types.ts +89 -0
package/dist/keyed.d.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Values kept by key, and waiters for keys that have no value yet: what both ends of a stream
|
|
3
|
+
* use to keep chunks by chunkId and wait for the ones still on their way.
|
|
4
|
+
*/
|
|
5
|
+
export declare class Keyed<T> {
|
|
6
|
+
#private;
|
|
7
|
+
get size(): number;
|
|
8
|
+
has(key: number): boolean;
|
|
9
|
+
get(key: number): T | undefined;
|
|
10
|
+
keys(): IterableIterator<number>;
|
|
11
|
+
/** Keeps `value` under `key` and wakes whoever waits for it. */
|
|
12
|
+
put(key: number, value: T): void;
|
|
13
|
+
delete(key: number): void;
|
|
14
|
+
/** The value of `key`: now if kept, else when it is put (or the whole set fails). */
|
|
15
|
+
wait(key: number): Promise<T>;
|
|
16
|
+
/** Answers every waiter for a key at or above `from` with `value` (the end of a stream). */
|
|
17
|
+
settleFrom(from: number, value: (key: number) => T): void;
|
|
18
|
+
/** Fails every waiter, now and later. */
|
|
19
|
+
fail(error: unknown): void;
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=keyed.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"keyed.d.ts","sourceRoot":"","sources":["../src/keyed.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,qBAAa,KAAK,CAAC,CAAC;;IAKlB,IAAI,IAAI,IAAI,MAAM,CAEjB;IACD,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAExB;IACD,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS,CAE9B;IACD,IAAI,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAE/B;IACD,gEAAgE;IAChE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,CAM/B;IACD,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAExB;IACD,qFAAqF;IACrF,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAQ5B;IACD,4FAA4F;IAC5F,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,CAAC,GAAG,IAAI,CAMxD;IACD,yCAAyC;IACzC,IAAI,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAIzB;CACF"}
|
package/dist/server.d.ts
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { FilesApi } from "@statewalker/webrun-files";
|
|
2
|
+
import type { FilesRpc } from "./types.js";
|
|
3
|
+
export interface FilesRpcServerOptions {
|
|
4
|
+
/** The file system served. */
|
|
5
|
+
fs: FilesApi;
|
|
6
|
+
/** Largest chunk a read stream answers. Default 256 KiB. */
|
|
7
|
+
chunkSize?: number;
|
|
8
|
+
/** Chunks produced ahead, counting from the highest one asked. Default 4. */
|
|
9
|
+
readAhead?: number;
|
|
10
|
+
/** A chunk is released once the reader asked for one `retain` chunks past it. Default 16. */
|
|
11
|
+
retain?: number;
|
|
12
|
+
/**
|
|
13
|
+
* A read request may name a chunk at most this many past the lowest chunk not yet delivered
|
|
14
|
+
* (INVALID beyond). Bounds what one stream makes the server pull and hold. Default 64.
|
|
15
|
+
*/
|
|
16
|
+
window?: number;
|
|
17
|
+
/**
|
|
18
|
+
* Chunks of a write kept ahead of the next one to write. Default 16. One window more may wait
|
|
19
|
+
* for room; a chunk beyond is refused (INVALID).
|
|
20
|
+
*/
|
|
21
|
+
writeWindow?: number;
|
|
22
|
+
/** Largest chunk a write takes. Default 4 MiB. */
|
|
23
|
+
maxChunkSize?: number;
|
|
24
|
+
/** A stream not used for this long is dropped (reads closed, writes aborted). Default 2 min. */
|
|
25
|
+
idleTimeout?: number;
|
|
26
|
+
/** Open streams at most. Default 256. */
|
|
27
|
+
maxStreams?: number;
|
|
28
|
+
/** Largest listing page. Default 256. */
|
|
29
|
+
maxPageSize?: number;
|
|
30
|
+
/** Sweeps idle streams every this many ms (default: only when a stream is opened). */
|
|
31
|
+
sweepInterval?: number;
|
|
32
|
+
/** The clock (tests). Default `Date.now`. */
|
|
33
|
+
now?: () => number;
|
|
34
|
+
}
|
|
35
|
+
/** The server: the RPC methods, plus maintenance. */
|
|
36
|
+
export interface FilesRpcServer extends FilesRpc {
|
|
37
|
+
/** Drops idle streams; returns how many. Opening a stream does it too. */
|
|
38
|
+
sweep(): number;
|
|
39
|
+
/** Streams open now. */
|
|
40
|
+
openStreams(): number;
|
|
41
|
+
/**
|
|
42
|
+
* Ends every stream (reads closed, writes aborted) and stops the sweep timer. It does not wait
|
|
43
|
+
* for the FilesApi's writes to settle, and waits at most a second for sources to end.
|
|
44
|
+
*/
|
|
45
|
+
close(): Promise<void>;
|
|
46
|
+
}
|
|
47
|
+
/** Serves a FilesApi as the FilesRpc methods. */
|
|
48
|
+
export declare function newFilesRpcServer(options: FilesRpcServerOptions): FilesRpcServer;
|
|
49
|
+
//# sourceMappingURL=server.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAY,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAGpE,OAAO,KAAK,EAAE,QAAQ,EAAsD,MAAM,YAAY,CAAC;AAE/F,MAAM,WAAW,qBAAqB;IACpC,8BAA8B;IAC9B,EAAE,EAAE,QAAQ,CAAC;IACb,4DAA4D;IAC5D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6EAA6E;IAC7E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6FAA6F;IAC7F,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kDAAkD;IAClD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gGAAgG;IAChG,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yCAAyC;IACzC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sFAAsF;IACtF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,6CAA6C;IAC7C,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED,qDAAqD;AACrD,MAAM,WAAW,cAAe,SAAQ,QAAQ;IAC9C,0EAA0E;IAC1E,KAAK,IAAI,MAAM,CAAC;IAChB,wBAAwB;IACxB,WAAW,IAAI,MAAM,CAAC;IACtB;;;OAGG;IACH,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAiGD,iDAAiD;AACjD,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,qBAAqB,GAAG,cAAc,CA2bhF"}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import type { FileInfo, FileStats } from "@statewalker/webrun-files";
|
|
2
|
+
/** One chunk of a read stream: its bytes, and whether it is the last. */
|
|
3
|
+
export interface ReadResult {
|
|
4
|
+
data: Uint8Array;
|
|
5
|
+
done: boolean;
|
|
6
|
+
}
|
|
7
|
+
/** One page of a listing. `more`: ask again with `after` = the last path of this page. */
|
|
8
|
+
export interface ListPage {
|
|
9
|
+
entries: FileInfo[];
|
|
10
|
+
more: boolean;
|
|
11
|
+
}
|
|
12
|
+
/** One page of `list`: `after` the given path (exclusive), at most `limit` entries. */
|
|
13
|
+
export interface ListPageOptions {
|
|
14
|
+
recursive?: boolean;
|
|
15
|
+
after?: string;
|
|
16
|
+
limit?: number;
|
|
17
|
+
}
|
|
18
|
+
/** Open options shared by reads and writes. */
|
|
19
|
+
export interface OpenOptions {
|
|
20
|
+
/**
|
|
21
|
+
* The stream's id, chosen by the caller (16 to 128 characters of `[A-Za-z0-9_-]`, random:
|
|
22
|
+
* `crypto.randomUUID()`), so that an open delivered twice opens one stream. Without it the
|
|
23
|
+
* server chooses one.
|
|
24
|
+
*/
|
|
25
|
+
streamId?: string;
|
|
26
|
+
}
|
|
27
|
+
export interface OpenReadOptions extends OpenOptions {
|
|
28
|
+
/** First byte (default 0). */
|
|
29
|
+
start?: number;
|
|
30
|
+
/** Bytes to read (default: to the end). */
|
|
31
|
+
length?: number;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* What the server exposes and the client calls: a FilesApi cut into calls an RPC can carry.
|
|
35
|
+
* Every argument and result is a plain value (strings, numbers, booleans, plain objects,
|
|
36
|
+
* Uint8Array), so any transport that passes structured-clonable values carries it: Durable
|
|
37
|
+
* Object and Workers RPC, postMessage, comlink, or a direct call.
|
|
38
|
+
*
|
|
39
|
+
* Content moves as chunks keyed by (streamId, chunkId). chunkId counts from 0 in each stream.
|
|
40
|
+
* Both sides keep chunks by their key and wait for the ones that have not arrived: the server
|
|
41
|
+
* answers `readChunk(id, k)` once chunk k exists, and writes chunk k once chunks 0..k-1 are
|
|
42
|
+
* written, whatever order they came in. Calls delivered twice are harmless.
|
|
43
|
+
*/
|
|
44
|
+
export interface FilesRpc {
|
|
45
|
+
stats(path: string): Promise<FileStats | undefined>;
|
|
46
|
+
exists(path: string): Promise<boolean>;
|
|
47
|
+
mkdir(path: string): Promise<void>;
|
|
48
|
+
remove(path: string): Promise<boolean>;
|
|
49
|
+
move(source: string, target: string): Promise<boolean>;
|
|
50
|
+
copy(source: string, target: string): Promise<boolean>;
|
|
51
|
+
list(path: string, options?: ListPageOptions): Promise<ListPage>;
|
|
52
|
+
/** Opens `path` for reading. A file that does not exist reads as nothing. */
|
|
53
|
+
openRead(path: string, options?: OpenReadOptions): Promise<{
|
|
54
|
+
streamId: string;
|
|
55
|
+
}>;
|
|
56
|
+
/**
|
|
57
|
+
* Chunk `chunkId` of the stream, once it exists. Past the end: empty and done. A chunk can be
|
|
58
|
+
* asked again (a retry) until the reader has moved `retain` chunks past it.
|
|
59
|
+
*/
|
|
60
|
+
readChunk(streamId: string, chunkId: number): Promise<ReadResult>;
|
|
61
|
+
/** Ends the stream (unknown or closed: nothing happens). */
|
|
62
|
+
closeRead(streamId: string): Promise<void>;
|
|
63
|
+
/** Opens `path` for writing; the FilesApi's `write` consumes the chunks in order. */
|
|
64
|
+
openWrite(path: string, options?: OpenOptions): Promise<{
|
|
65
|
+
streamId: string;
|
|
66
|
+
}>;
|
|
67
|
+
/**
|
|
68
|
+
* Hands over chunk `chunkId`. Resolves once it is kept (not yet written): at once if it is
|
|
69
|
+
* within `writeWindow` chunks of the next to write, else when it is. A chunk already received
|
|
70
|
+
* is ignored.
|
|
71
|
+
*/
|
|
72
|
+
writeChunk(streamId: string, chunkId: number, data: Uint8Array | ArrayBuffer): Promise<void>;
|
|
73
|
+
/** The stream had `chunks` chunks: resolves when all of them are written. */
|
|
74
|
+
closeWrite(streamId: string, chunks: number): Promise<void>;
|
|
75
|
+
/** Cancels the write (unknown or closed: nothing happens). */
|
|
76
|
+
abortWrite(streamId: string, reason?: string): Promise<void>;
|
|
77
|
+
}
|
|
78
|
+
/** Error codes of FilesRpcError. */
|
|
79
|
+
export type FilesRpcErrorCode = "UNKNOWN_STREAM" | "CHUNK_RELEASED" | "TOO_MANY_STREAMS" | "ABORTED" | "INVALID";
|
|
80
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAErE,yEAAyE;AACzE,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,OAAO,CAAC;CACf;AAED,0FAA0F;AAC1F,MAAM,WAAW,QAAQ;IACvB,OAAO,EAAE,QAAQ,EAAE,CAAC;IACpB,IAAI,EAAE,OAAO,CAAC;CACf;AAED,uFAAuF;AACvF,MAAM,WAAW,eAAe;IAC9B,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,+CAA+C;AAC/C,MAAM,WAAW,WAAW;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAgB,SAAQ,WAAW;IAClD,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2CAA2C;IAC3C,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,QAAQ;IACvB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC,CAAC;IACpD,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACvC,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACvC,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACvD,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACvD,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEjE,6EAA6E;IAC7E,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjF;;;OAGG;IACH,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAClE,4DAA4D;IAC5D,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE3C,qFAAqF;IACrF,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9E;;;;OAIG;IACH,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7F,6EAA6E;IAC7E,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5D,8DAA8D;IAC9D,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9D;AAED,oCAAoC;AACpC,MAAM,MAAM,iBAAiB,GACzB,gBAAgB,GAChB,gBAAgB,GAChB,kBAAkB,GAClB,SAAS,GACT,SAAS,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,57 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@statewalker/webrun-files-rpc",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"private": false,
|
|
5
|
+
"type": "module",
|
|
6
|
+
"description": "FilesApi over any RPC: streamed reads and writes as chunks kept and awaited by (streamId, chunkId)",
|
|
7
|
+
"bugs": {
|
|
8
|
+
"url": "https://github.com/statewalker/webrun-files/issues"
|
|
9
|
+
},
|
|
10
|
+
"homepage": "https://github.com/statewalker/webrun-files/tree/main/packages/webrun-files-rpc#readme",
|
|
11
|
+
"author": {
|
|
12
|
+
"name": "Mikhail Kotelnikov",
|
|
13
|
+
"email": "mikhail.kotelnikov@gmail.com"
|
|
14
|
+
},
|
|
15
|
+
"license": "MIT",
|
|
16
|
+
"repository": {
|
|
17
|
+
"type": "git",
|
|
18
|
+
"url": "git+https://github.com/statewalker/webrun-files.git",
|
|
19
|
+
"directory": "packages/webrun-files-rpc"
|
|
20
|
+
},
|
|
21
|
+
"main": "./dist/cjs/index.cjs",
|
|
22
|
+
"module": "./dist/esm/index.js",
|
|
23
|
+
"types": "./dist/index.d.ts",
|
|
24
|
+
"exports": {
|
|
25
|
+
".": {
|
|
26
|
+
"source": "./src/index.ts",
|
|
27
|
+
"types": "./dist/index.d.ts",
|
|
28
|
+
"import": "./dist/esm/index.js",
|
|
29
|
+
"require": "./dist/cjs/index.cjs"
|
|
30
|
+
}
|
|
31
|
+
},
|
|
32
|
+
"files": [
|
|
33
|
+
"dist",
|
|
34
|
+
"src"
|
|
35
|
+
],
|
|
36
|
+
"dependencies": {
|
|
37
|
+
"@statewalker/webrun-files": "^0.10.4"
|
|
38
|
+
},
|
|
39
|
+
"devDependencies": {
|
|
40
|
+
"@types/node": "^26.6.4",
|
|
41
|
+
"rimraf": "^6.1.3",
|
|
42
|
+
"rolldown": "^1.2.12",
|
|
43
|
+
"typescript": "^7.0.2",
|
|
44
|
+
"vitest": "^4.1.4 || ^5.0.3",
|
|
45
|
+
"@statewalker/webrun-files-mem": "^0.10.4",
|
|
46
|
+
"@statewalker/webrun-files-tests": "^3.0.0"
|
|
47
|
+
},
|
|
48
|
+
"sideEffects": false,
|
|
49
|
+
"publishConfig": {
|
|
50
|
+
"access": "public"
|
|
51
|
+
},
|
|
52
|
+
"scripts": {
|
|
53
|
+
"build": "rimraf dist && rolldown -c && tsc --emitDeclarationOnly --declaration",
|
|
54
|
+
"test": "vitest run",
|
|
55
|
+
"lint": "biome lint src tests"
|
|
56
|
+
}
|
|
6
57
|
}
|
package/src/client.ts
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
FileInfo,
|
|
3
|
+
FileStats,
|
|
4
|
+
FilesApi,
|
|
5
|
+
ListOptions,
|
|
6
|
+
ReadOptions,
|
|
7
|
+
} from "@statewalker/webrun-files";
|
|
8
|
+
import type { FilesRpc, ReadResult } from "./types.js";
|
|
9
|
+
|
|
10
|
+
export interface FilesRpcClientOptions {
|
|
11
|
+
/** The server's methods, through any transport (a stub, a proxy, the server itself). */
|
|
12
|
+
rpc: FilesRpc;
|
|
13
|
+
/** Bytes per chunk written. Default 256 KiB (keep it at most the server's `maxChunkSize`). */
|
|
14
|
+
chunkSize?: number;
|
|
15
|
+
/** Chunk requests kept in flight while reading. Default 4 (below the server's `window`). */
|
|
16
|
+
readAhead?: number;
|
|
17
|
+
/** Chunks sent and not yet acknowledged while writing. Default 4. */
|
|
18
|
+
writeAhead?: number;
|
|
19
|
+
/** Listing page size. Default 256. */
|
|
20
|
+
pageSize?: number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const toBytes = (v: unknown): Uint8Array =>
|
|
24
|
+
v instanceof Uint8Array
|
|
25
|
+
? v
|
|
26
|
+
: ArrayBuffer.isView(v)
|
|
27
|
+
? new Uint8Array(v.buffer, v.byteOffset, v.byteLength)
|
|
28
|
+
: new Uint8Array(v as ArrayBuffer);
|
|
29
|
+
|
|
30
|
+
const newStreamId = () => crypto.randomUUID().replace(/-/g, "");
|
|
31
|
+
|
|
32
|
+
/** Cuts any sequence of byte arrays into chunks of exactly `size` bytes (the last may be shorter). */
|
|
33
|
+
async function* rechunk(
|
|
34
|
+
content: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,
|
|
35
|
+
size: number,
|
|
36
|
+
): AsyncGenerator<Uint8Array> {
|
|
37
|
+
let carry: Uint8Array | undefined;
|
|
38
|
+
let carried = 0;
|
|
39
|
+
for await (const piece of content as AsyncIterable<Uint8Array>) {
|
|
40
|
+
let part = toBytes(piece);
|
|
41
|
+
while (part.length) {
|
|
42
|
+
if (!carry && part.length >= size) {
|
|
43
|
+
yield part.slice(0, size);
|
|
44
|
+
part = part.subarray(size);
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
carry ??= new Uint8Array(size);
|
|
48
|
+
const n = Math.min(size - carried, part.length);
|
|
49
|
+
carry.set(part.subarray(0, n), carried);
|
|
50
|
+
carried += n;
|
|
51
|
+
part = part.subarray(n);
|
|
52
|
+
if (carried === size) {
|
|
53
|
+
yield carry;
|
|
54
|
+
carry = undefined;
|
|
55
|
+
carried = 0;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
if (carry && carried) yield carry.slice(0, carried);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** A FilesApi whose every call goes through a FilesRpc; content moves in keyed chunks. */
|
|
63
|
+
export class FilesRpcClient implements FilesApi {
|
|
64
|
+
#rpc: FilesRpc;
|
|
65
|
+
#chunkSize: number;
|
|
66
|
+
#readAhead: number;
|
|
67
|
+
#writeAhead: number;
|
|
68
|
+
#pageSize: number;
|
|
69
|
+
|
|
70
|
+
constructor(options: FilesRpcClientOptions) {
|
|
71
|
+
this.#rpc = options.rpc;
|
|
72
|
+
this.#chunkSize = Math.max(1, options.chunkSize ?? 256 * 1024);
|
|
73
|
+
this.#readAhead = Math.max(1, options.readAhead ?? 4);
|
|
74
|
+
this.#writeAhead = Math.max(1, options.writeAhead ?? 4);
|
|
75
|
+
this.#pageSize = Math.max(1, options.pageSize ?? 256);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Asks for chunks 0, 1, 2, … keeping `readAhead` requests in flight, and yields them in key
|
|
80
|
+
* order whatever order the answers come in.
|
|
81
|
+
*/
|
|
82
|
+
async *read(path: string, options: ReadOptions = {}): AsyncIterable<Uint8Array> {
|
|
83
|
+
const { start, length, signal } = options;
|
|
84
|
+
if (length === 0) return;
|
|
85
|
+
signal?.throwIfAborted();
|
|
86
|
+
const { streamId } = await this.#rpc.openRead(path, {
|
|
87
|
+
start,
|
|
88
|
+
length,
|
|
89
|
+
streamId: newStreamId(),
|
|
90
|
+
});
|
|
91
|
+
const pending = new Map<number, Promise<ReadResult>>();
|
|
92
|
+
// An abort interrupts the wait for an answer, not only the loop between answers.
|
|
93
|
+
const aborted = signal
|
|
94
|
+
? new Promise<never>((_, reject) => {
|
|
95
|
+
const fail = () => reject(signal.reason ?? new Error("aborted"));
|
|
96
|
+
if (signal.aborted) fail();
|
|
97
|
+
else signal.addEventListener("abort", fail, { once: true });
|
|
98
|
+
})
|
|
99
|
+
: undefined;
|
|
100
|
+
aborted?.catch(() => {});
|
|
101
|
+
let asked = 0;
|
|
102
|
+
let finished = false;
|
|
103
|
+
try {
|
|
104
|
+
for (let next = 0; ; next++) {
|
|
105
|
+
while (!finished && asked < next + this.#readAhead) {
|
|
106
|
+
const request = this.#rpc.readChunk(streamId, asked);
|
|
107
|
+
request.catch(() => {}); // awaited below, in order
|
|
108
|
+
pending.set(asked++, request);
|
|
109
|
+
}
|
|
110
|
+
const answer = pending.get(next) as Promise<ReadResult>;
|
|
111
|
+
pending.delete(next);
|
|
112
|
+
const { data, done } = await (aborted ? Promise.race([answer, aborted]) : answer);
|
|
113
|
+
signal?.throwIfAborted();
|
|
114
|
+
if (done) finished = true;
|
|
115
|
+
const bytes = toBytes(data);
|
|
116
|
+
if (bytes.length) yield bytes;
|
|
117
|
+
if (done) return;
|
|
118
|
+
signal?.throwIfAborted();
|
|
119
|
+
}
|
|
120
|
+
} finally {
|
|
121
|
+
await this.#rpc.closeRead(streamId).catch(() => {});
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Sends chunks of `chunkSize` bytes keyed 0, 1, 2, … with up to `writeAhead` unacknowledged,
|
|
127
|
+
* then `closeWrite` with their count; any failure aborts the write.
|
|
128
|
+
*/
|
|
129
|
+
async write(
|
|
130
|
+
path: string,
|
|
131
|
+
content: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,
|
|
132
|
+
): Promise<void> {
|
|
133
|
+
const { streamId } = await this.#rpc.openWrite(path, { streamId: newStreamId() });
|
|
134
|
+
const inFlight = new Set<Promise<void>>();
|
|
135
|
+
let failure: { error: unknown } | undefined;
|
|
136
|
+
let count = 0;
|
|
137
|
+
try {
|
|
138
|
+
for await (const chunk of rechunk(content, this.#chunkSize)) {
|
|
139
|
+
if (failure) throw failure.error;
|
|
140
|
+
const sent: Promise<void> = this.#rpc.writeChunk(streamId, count++, chunk).then(
|
|
141
|
+
() => {
|
|
142
|
+
inFlight.delete(sent);
|
|
143
|
+
},
|
|
144
|
+
(error) => {
|
|
145
|
+
inFlight.delete(sent);
|
|
146
|
+
failure ??= { error };
|
|
147
|
+
},
|
|
148
|
+
);
|
|
149
|
+
inFlight.add(sent);
|
|
150
|
+
while (inFlight.size >= this.#writeAhead) await Promise.race(inFlight);
|
|
151
|
+
}
|
|
152
|
+
await Promise.all(inFlight);
|
|
153
|
+
if (failure) throw failure.error;
|
|
154
|
+
await this.#rpc.closeWrite(streamId, count);
|
|
155
|
+
} catch (e) {
|
|
156
|
+
await this.#rpc.abortWrite(streamId, String((e as Error)?.message ?? e)).catch(() => {});
|
|
157
|
+
throw e;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
async mkdir(path: string): Promise<void> {
|
|
162
|
+
await this.#rpc.mkdir(path);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** Pages of `pageSize`, each asked after the last path of the one before. */
|
|
166
|
+
async *list(path: string, options: ListOptions = {}): AsyncIterable<FileInfo> {
|
|
167
|
+
let after = options.after;
|
|
168
|
+
for (;;) {
|
|
169
|
+
const page = await this.#rpc.list(path, {
|
|
170
|
+
recursive: options.recursive,
|
|
171
|
+
after,
|
|
172
|
+
limit: this.#pageSize,
|
|
173
|
+
});
|
|
174
|
+
yield* page.entries;
|
|
175
|
+
if (!page.more || page.entries.length === 0) return;
|
|
176
|
+
after = page.entries[page.entries.length - 1]?.path;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
stats(path: string): Promise<FileStats | undefined> {
|
|
181
|
+
return this.#rpc.stats(path);
|
|
182
|
+
}
|
|
183
|
+
exists(path: string): Promise<boolean> {
|
|
184
|
+
return this.#rpc.exists(path);
|
|
185
|
+
}
|
|
186
|
+
remove(path: string): Promise<boolean> {
|
|
187
|
+
return this.#rpc.remove(path);
|
|
188
|
+
}
|
|
189
|
+
move(source: string, target: string): Promise<boolean> {
|
|
190
|
+
return this.#rpc.move(source, target);
|
|
191
|
+
}
|
|
192
|
+
copy(source: string, target: string): Promise<boolean> {
|
|
193
|
+
return this.#rpc.copy(source, target);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** A FilesApi over a FilesRpc. */
|
|
198
|
+
export function newFilesRpcClient(options: FilesRpcClientOptions): FilesRpcClient {
|
|
199
|
+
return new FilesRpcClient(options);
|
|
200
|
+
}
|
package/src/errors.ts
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { FilesRpcErrorCode } from "./types.js";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* An error of the protocol itself. The code also leads the message (`UNKNOWN_STREAM: …`), since
|
|
5
|
+
* a structured clone keeps an error's name and message but not its other properties:
|
|
6
|
+
* `errorCode(e)` reads it back on the other side of a transport.
|
|
7
|
+
*/
|
|
8
|
+
export class FilesRpcError extends Error {
|
|
9
|
+
readonly code: FilesRpcErrorCode;
|
|
10
|
+
constructor(code: FilesRpcErrorCode, message: string) {
|
|
11
|
+
super(`${code}: ${message}`);
|
|
12
|
+
this.name = "FilesRpcError";
|
|
13
|
+
this.code = code;
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** The FilesRpcError code of an error, also after it crossed a transport; undefined otherwise. */
|
|
18
|
+
export function errorCode(e: unknown): FilesRpcErrorCode | undefined {
|
|
19
|
+
const code = (e as { code?: unknown })?.code;
|
|
20
|
+
if (typeof code === "string") return code as FilesRpcErrorCode;
|
|
21
|
+
const m = /^(UNKNOWN_STREAM|CHUNK_RELEASED|TOO_MANY_STREAMS|ABORTED|INVALID):/.exec(
|
|
22
|
+
String((e as Error)?.message ?? ""),
|
|
23
|
+
);
|
|
24
|
+
return m?.[1] as FilesRpcErrorCode | undefined;
|
|
25
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* webrun-files-rpc: a FilesApi over any RPC. The server serves a FilesApi as plain calls; the
|
|
3
|
+
* client is a FilesApi over them. Content is streamed as chunks keyed by (streamId, chunkId):
|
|
4
|
+
* both sides keep chunks by their key and wait for those still on their way, so calls may arrive
|
|
5
|
+
* out of order, concurrently, or twice.
|
|
6
|
+
*/
|
|
7
|
+
export { FilesRpcClient, type FilesRpcClientOptions, newFilesRpcClient } from "./client.js";
|
|
8
|
+
export { errorCode, FilesRpcError } from "./errors.js";
|
|
9
|
+
export { type FilesRpcServer, type FilesRpcServerOptions, newFilesRpcServer } from "./server.js";
|
|
10
|
+
export type {
|
|
11
|
+
FilesRpc,
|
|
12
|
+
FilesRpcErrorCode,
|
|
13
|
+
ListPage,
|
|
14
|
+
ListPageOptions,
|
|
15
|
+
OpenOptions,
|
|
16
|
+
OpenReadOptions,
|
|
17
|
+
ReadResult,
|
|
18
|
+
} from "./types.js";
|
package/src/keyed.ts
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Values kept by key, and waiters for keys that have no value yet: what both ends of a stream
|
|
3
|
+
* use to keep chunks by chunkId and wait for the ones still on their way.
|
|
4
|
+
*/
|
|
5
|
+
export class Keyed<T> {
|
|
6
|
+
#values = new Map<number, T>();
|
|
7
|
+
#waiters = new Map<number, { resolve: (v: T) => void; reject: (e: unknown) => void }[]>();
|
|
8
|
+
#failure: { error: unknown } | undefined;
|
|
9
|
+
|
|
10
|
+
get size(): number {
|
|
11
|
+
return this.#values.size;
|
|
12
|
+
}
|
|
13
|
+
has(key: number): boolean {
|
|
14
|
+
return this.#values.has(key);
|
|
15
|
+
}
|
|
16
|
+
get(key: number): T | undefined {
|
|
17
|
+
return this.#values.get(key);
|
|
18
|
+
}
|
|
19
|
+
keys(): IterableIterator<number> {
|
|
20
|
+
return this.#values.keys();
|
|
21
|
+
}
|
|
22
|
+
/** Keeps `value` under `key` and wakes whoever waits for it. */
|
|
23
|
+
put(key: number, value: T): void {
|
|
24
|
+
this.#values.set(key, value);
|
|
25
|
+
const waiting = this.#waiters.get(key);
|
|
26
|
+
if (!waiting) return;
|
|
27
|
+
this.#waiters.delete(key);
|
|
28
|
+
for (const w of waiting) w.resolve(value);
|
|
29
|
+
}
|
|
30
|
+
delete(key: number): void {
|
|
31
|
+
this.#values.delete(key);
|
|
32
|
+
}
|
|
33
|
+
/** The value of `key`: now if kept, else when it is put (or the whole set fails). */
|
|
34
|
+
wait(key: number): Promise<T> {
|
|
35
|
+
if (this.#values.has(key)) return Promise.resolve(this.#values.get(key) as T);
|
|
36
|
+
if (this.#failure) return Promise.reject(this.#failure.error);
|
|
37
|
+
return new Promise<T>((resolve, reject) => {
|
|
38
|
+
const list = this.#waiters.get(key);
|
|
39
|
+
if (list) list.push({ resolve, reject });
|
|
40
|
+
else this.#waiters.set(key, [{ resolve, reject }]);
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
/** Answers every waiter for a key at or above `from` with `value` (the end of a stream). */
|
|
44
|
+
settleFrom(from: number, value: (key: number) => T): void {
|
|
45
|
+
for (const [key, waiting] of [...this.#waiters]) {
|
|
46
|
+
if (key < from) continue;
|
|
47
|
+
this.#waiters.delete(key);
|
|
48
|
+
for (const w of waiting) w.resolve(value(key));
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
/** Fails every waiter, now and later. */
|
|
52
|
+
fail(error: unknown): void {
|
|
53
|
+
this.#failure ??= { error };
|
|
54
|
+
for (const waiting of this.#waiters.values()) for (const w of waiting) w.reject(error);
|
|
55
|
+
this.#waiters.clear();
|
|
56
|
+
}
|
|
57
|
+
}
|