@ricsam/r5d-worker 0.0.167 → 0.0.168
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/package.json +1 -1
- package/dist/mjs/internal-r5dctl.cjs +1 -1
- package/dist/mjs/main.mjs +2 -2
- package/dist/mjs/package.json +1 -1
- package/dist/mjs/runtime/workspace/authority.mjs +4 -14
- package/dist/mjs/runtime/workspace/storage-client.mjs +57 -9
- package/dist/mjs/runtime/workspace/storage-wire.mjs +3 -1
- package/dist/types/runtime/workspace/storage-client.d.ts +35 -1
- package/dist/types/runtime/workspace/storage-wire.d.ts +7 -0
- package/package.json +2 -2
package/dist/cjs/package.json
CHANGED
|
@@ -20605,7 +20605,7 @@ function resolveEntrypointPath(entrypoint) {
|
|
|
20605
20605
|
}
|
|
20606
20606
|
}
|
|
20607
20607
|
function getR5dctlVersion() {
|
|
20608
|
-
if (true) return "0.0.
|
|
20608
|
+
if (true) return "0.0.168";
|
|
20609
20609
|
const entrypoint = process.argv[1] ? resolveEntrypointPath(process.argv[1]) : null;
|
|
20610
20610
|
let current = entrypoint ? import_node_path2.default.dirname(entrypoint) : process.cwd();
|
|
20611
20611
|
for (let index = 0; index < 12; index += 1) {
|
package/dist/mjs/main.mjs
CHANGED
|
@@ -7,7 +7,7 @@ import { startManagerRpc } from "./runtime/releases/rpc-main.mjs";
|
|
|
7
7
|
import { ManagerRpcConfig } from "./runtime/releases/rpc-protocol.mjs";
|
|
8
8
|
const args = process.argv.slice(2);
|
|
9
9
|
if (args.includes("--version")) {
|
|
10
|
-
console.log(`r5d-worker ${true ? "0.0.
|
|
10
|
+
console.log(`r5d-worker ${true ? "0.0.168" : "development"}`);
|
|
11
11
|
} else if (!args.length || args.includes("--help")) {
|
|
12
12
|
console.log(
|
|
13
13
|
"Usage: r5d-worker start --label <label> [--root <dir>] [--base-url <url>] [--token <worker-token>]\n r5d-worker executor /absolute/private/config.json\n r5d-worker manager /absolute/private/config.json\n\nRun independently supervised durable runtime services. Provision configuration with r5dinfra."
|
|
@@ -15,7 +15,7 @@ if (args.includes("--version")) {
|
|
|
15
15
|
} else if (args[0] === "start") {
|
|
16
16
|
const runtime = await startPersonalWorker(
|
|
17
17
|
parsePersonalWorkerOptions(args.slice(1)),
|
|
18
|
-
true ? "0.0.
|
|
18
|
+
true ? "0.0.168" : "development"
|
|
19
19
|
);
|
|
20
20
|
console.log(`Worker connected: ${runtime.resourceId}`);
|
|
21
21
|
let closing = false;
|
package/dist/mjs/package.json
CHANGED
|
@@ -1328,16 +1328,17 @@ class WorkspaceAuthority {
|
|
|
1328
1328
|
async downloadBundle(storage, repositoryId) {
|
|
1329
1329
|
const chunks = [];
|
|
1330
1330
|
let offset = 0, snapshot = null, size = -1, hash = "";
|
|
1331
|
+
const chunkBytes = await storage.chunkBytes();
|
|
1331
1332
|
do {
|
|
1332
1333
|
const chunk = await storage.read({
|
|
1333
1334
|
method: "repository.bundle",
|
|
1334
1335
|
repositoryId,
|
|
1335
1336
|
snapshot,
|
|
1336
1337
|
offset,
|
|
1337
|
-
limit:
|
|
1338
|
+
limit: chunkBytes
|
|
1338
1339
|
});
|
|
1339
1340
|
const bytes = Buffer.from(chunk.data, "base64");
|
|
1340
|
-
if (chunk.data !== bytes.toString("base64") || bytes.length >
|
|
1341
|
+
if (chunk.data !== bytes.toString("base64") || bytes.length > chunkBytes || chunk.offset !== offset || chunk.nextOffset !== offset + bytes.length || !bytes.length || chunk.size > STORAGE_LIMITS.blobBytes || chunk.nextOffset > chunk.size || snapshot && (snapshot !== chunk.snapshot || size !== chunk.size || hash !== chunk.sha256))
|
|
1341
1342
|
throw new WorkspaceError("invalid_bundle", "Chunk identity/size/offset changed; no checkout performed");
|
|
1342
1343
|
snapshot = chunk.snapshot;
|
|
1343
1344
|
size = chunk.size;
|
|
@@ -1350,18 +1351,7 @@ class WorkspaceAuthority {
|
|
|
1350
1351
|
return bundle;
|
|
1351
1352
|
}
|
|
1352
1353
|
async uploadBundle(storage, bytes, blobId, operationId) {
|
|
1353
|
-
|
|
1354
|
-
const end = Math.min(bytes.length, offset + STORAGE_LIMITS.chunkBytes);
|
|
1355
|
-
await storage.mutate({
|
|
1356
|
-
method: "blob.append",
|
|
1357
|
-
operationId: `${operationId}-chunk-${offset}`,
|
|
1358
|
-
blobId,
|
|
1359
|
-
kind: "blob",
|
|
1360
|
-
offset,
|
|
1361
|
-
data: bytes.subarray(offset, end).toString("base64"),
|
|
1362
|
-
seal: end === bytes.length
|
|
1363
|
-
});
|
|
1364
|
-
}
|
|
1354
|
+
await storage.appendBlob(bytes, blobId, operationId);
|
|
1365
1355
|
}
|
|
1366
1356
|
async unbundleInto(b, bytes) {
|
|
1367
1357
|
const file = path.join(b.directory, `${randomUUID()}.bundle`);
|
|
@@ -1,17 +1,24 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { OwnershipFence } from "@ricsam/r5d-api/runtime-protocol";
|
|
3
|
-
import { STORAGE_LIMITS, StorageRequest } from "./storage-wire.mjs";
|
|
3
|
+
import { LEGACY_CHUNK_BYTES, STORAGE_LIMITS, StorageRequest } from "./storage-wire.mjs";
|
|
4
4
|
import { WorkspaceError } from "./contracts.mjs";
|
|
5
5
|
import { readBoundedBody } from "./http-body.mjs";
|
|
6
6
|
class WorkspaceStorageClient {
|
|
7
|
-
constructor(installationId, transport, caller) {
|
|
7
|
+
constructor(installationId, transport, caller, helloTtlMs = 1e4) {
|
|
8
8
|
this.installationId = installationId;
|
|
9
9
|
this.transport = transport;
|
|
10
10
|
this.caller = caller;
|
|
11
|
+
this.helloTtlMs = helloTtlMs;
|
|
11
12
|
}
|
|
12
13
|
installationId;
|
|
13
14
|
transport;
|
|
14
15
|
caller;
|
|
16
|
+
helloTtlMs;
|
|
17
|
+
/** The authority's greeting (fence, capabilities, limits) is reused for a
|
|
18
|
+
* short window: every mutation used to pay a second round-trip for it, and a
|
|
19
|
+
* fence that goes stale within the window is refused before admission by the
|
|
20
|
+
* authority itself. Any transport failure discards it. */
|
|
21
|
+
greeting;
|
|
15
22
|
static http(options) {
|
|
16
23
|
const url = new URL(options.url);
|
|
17
24
|
if (url.username || url.password || url.search || url.hash || url.pathname !== "/internal/storage/v1/rpc" || url.protocol !== "https:" && !(url.protocol === "http:" && ["127.0.0.1", "[::1]"].includes(url.hostname)))
|
|
@@ -41,19 +48,60 @@ class WorkspaceStorageClient {
|
|
|
41
48
|
read(request) {
|
|
42
49
|
return this.transport(StorageRequest.parse(request));
|
|
43
50
|
}
|
|
51
|
+
/** The authority's current greeting, fetched at most once per TTL. */
|
|
52
|
+
async hello() {
|
|
53
|
+
if (this.greeting && Date.now() - this.greeting.at < this.helloTtlMs) return this.greeting.value;
|
|
54
|
+
const value = StorageHello.parse(await this.read({ method: "hello" }));
|
|
55
|
+
if (value.installationId !== this.installationId) throw new WorkspaceError("wrong_installation", "Storage authority belongs to another installation");
|
|
56
|
+
this.greeting = { value, at: Date.now() };
|
|
57
|
+
return value;
|
|
58
|
+
}
|
|
59
|
+
/** Bytes per blob chunk: this client's ceiling, negotiated down to what the
|
|
60
|
+
* authority advertises (an authority predating the advertisement gets the
|
|
61
|
+
* legacy size). */
|
|
62
|
+
async chunkBytes() {
|
|
63
|
+
const advertised = (await this.hello()).limits?.chunkBytes ?? LEGACY_CHUNK_BYTES;
|
|
64
|
+
return Math.max(1, Math.min(STORAGE_LIMITS.chunkBytes, advertised));
|
|
65
|
+
}
|
|
44
66
|
async mutate(input) {
|
|
45
|
-
const hello =
|
|
46
|
-
protocol: z.literal(1),
|
|
47
|
-
installationId: z.literal(this.installationId),
|
|
48
|
-
fence: OwnershipFence,
|
|
49
|
-
capabilities: z.array(z.string())
|
|
50
|
-
}).passthrough().parse(await this.read({ method: "hello" }));
|
|
67
|
+
const hello = await this.hello();
|
|
51
68
|
const callerFence = OwnershipFence.parse(await this.caller());
|
|
52
69
|
if (hello.fence.installationId !== this.installationId || callerFence.installationId !== this.installationId || callerFence.resourceType === "storage-authority")
|
|
53
70
|
throw new WorkspaceError("wrong_installation", "Separate installation-bound caller ownership required");
|
|
54
|
-
|
|
71
|
+
try {
|
|
72
|
+
return await this.transport(StorageRequest.parse({ ...input, fence: hello.fence, callerFence }));
|
|
73
|
+
} catch (error) {
|
|
74
|
+
this.greeting = void 0;
|
|
75
|
+
throw error;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
/** Uploads `bytes` as blob `blobId` in ordered, negotiated-size chunks; the
|
|
79
|
+
* last append seals it. Each chunk is its own durable operation
|
|
80
|
+
* (`<operationId>-chunk-<offset>`), so a repeated upload of the same bytes
|
|
81
|
+
* settles against the original receipts. */
|
|
82
|
+
async appendBlob(bytes, blobId, operationId) {
|
|
83
|
+
const chunkBytes = await this.chunkBytes();
|
|
84
|
+
for (let offset = 0; offset < bytes.length; offset += chunkBytes) {
|
|
85
|
+
const end = Math.min(bytes.length, offset + chunkBytes);
|
|
86
|
+
await this.mutate({
|
|
87
|
+
method: "blob.append",
|
|
88
|
+
operationId: `${operationId}-chunk-${offset}`,
|
|
89
|
+
blobId,
|
|
90
|
+
kind: "blob",
|
|
91
|
+
offset,
|
|
92
|
+
data: bytes.subarray(offset, end).toString("base64"),
|
|
93
|
+
seal: end === bytes.length
|
|
94
|
+
});
|
|
95
|
+
}
|
|
55
96
|
}
|
|
56
97
|
}
|
|
98
|
+
const StorageHello = z.object({
|
|
99
|
+
protocol: z.literal(1),
|
|
100
|
+
installationId: z.string(),
|
|
101
|
+
fence: OwnershipFence,
|
|
102
|
+
capabilities: z.array(z.string()),
|
|
103
|
+
limits: z.object({ chunkBytes: z.number().int().min(1).optional() }).passthrough().optional()
|
|
104
|
+
}).passthrough();
|
|
57
105
|
export {
|
|
58
106
|
WorkspaceStorageClient
|
|
59
107
|
};
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { OwnershipFence, RuntimeId } from "@ricsam/r5d-api/runtime-protocol";
|
|
3
|
-
const STORAGE_LIMITS = { chunkBytes:
|
|
3
|
+
const STORAGE_LIMITS = { chunkBytes: 1024 * 1024, blobBytes: 128 * 1024 * 1024, requestBytes: 1536 * 1024, treeEntries: 2e5 };
|
|
4
|
+
const LEGACY_CHUNK_BYTES = 64 * 1024;
|
|
4
5
|
const StorageId = z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]{0,99}$/);
|
|
5
6
|
const GitOid = z.string().regex(/^[0-9a-f]{40}$/);
|
|
6
7
|
const BranchName = z.string().min(1).max(150).refine((value) => value.split("/").every((part) => /^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(part)));
|
|
@@ -49,6 +50,7 @@ function safeTreePath(value) {
|
|
|
49
50
|
export {
|
|
50
51
|
BranchName,
|
|
51
52
|
GitOid,
|
|
53
|
+
LEGACY_CHUNK_BYTES,
|
|
52
54
|
STORAGE_LIMITS,
|
|
53
55
|
StorageId,
|
|
54
56
|
StorageRequest,
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
1
2
|
import { OwnershipFence } from "@ricsam/r5d-api/runtime-protocol";
|
|
2
3
|
import { StorageRequest, type StorageMutation } from "./storage-wire";
|
|
3
4
|
export type StorageTransport = (request: StorageRequest) => Promise<unknown>;
|
|
@@ -9,7 +10,13 @@ export declare class WorkspaceStorageClient {
|
|
|
9
10
|
readonly installationId: string;
|
|
10
11
|
private readonly transport;
|
|
11
12
|
private readonly caller;
|
|
12
|
-
|
|
13
|
+
private readonly helloTtlMs;
|
|
14
|
+
/** The authority's greeting (fence, capabilities, limits) is reused for a
|
|
15
|
+
* short window: every mutation used to pay a second round-trip for it, and a
|
|
16
|
+
* fence that goes stale within the window is refused before admission by the
|
|
17
|
+
* authority itself. Any transport failure discards it. */
|
|
18
|
+
private greeting?;
|
|
19
|
+
constructor(installationId: string, transport: StorageTransport, caller: () => Promise<OwnershipFence>, helloTtlMs?: number);
|
|
13
20
|
static http(options: {
|
|
14
21
|
installationId: string;
|
|
15
22
|
url: string;
|
|
@@ -18,6 +25,33 @@ export declare class WorkspaceStorageClient {
|
|
|
18
25
|
fetch?: typeof fetch;
|
|
19
26
|
}): WorkspaceStorageClient;
|
|
20
27
|
read<T>(request: Exclude<StorageRequest, StorageMutation>): Promise<T>;
|
|
28
|
+
/** The authority's current greeting, fetched at most once per TTL. */
|
|
29
|
+
hello(): Promise<StorageHello>;
|
|
30
|
+
/** Bytes per blob chunk: this client's ceiling, negotiated down to what the
|
|
31
|
+
* authority advertises (an authority predating the advertisement gets the
|
|
32
|
+
* legacy size). */
|
|
33
|
+
chunkBytes(): Promise<number>;
|
|
21
34
|
mutate<T>(input: MutationInput): Promise<T>;
|
|
35
|
+
/** Uploads `bytes` as blob `blobId` in ordered, negotiated-size chunks; the
|
|
36
|
+
* last append seals it. Each chunk is its own durable operation
|
|
37
|
+
* (`<operationId>-chunk-<offset>`), so a repeated upload of the same bytes
|
|
38
|
+
* settles against the original receipts. */
|
|
39
|
+
appendBlob(bytes: Buffer, blobId: string, operationId: string): Promise<void>;
|
|
22
40
|
}
|
|
41
|
+
declare const StorageHello: z.ZodObject<{
|
|
42
|
+
protocol: z.ZodLiteral<1>;
|
|
43
|
+
installationId: z.ZodString;
|
|
44
|
+
fence: z.ZodObject<{
|
|
45
|
+
installationId: z.ZodString;
|
|
46
|
+
resourceType: z.ZodString;
|
|
47
|
+
resourceId: z.ZodString;
|
|
48
|
+
ownerId: z.ZodString;
|
|
49
|
+
epoch: z.ZodNumber;
|
|
50
|
+
}, z.core.$strict>;
|
|
51
|
+
capabilities: z.ZodArray<z.ZodString>;
|
|
52
|
+
limits: z.ZodOptional<z.ZodObject<{
|
|
53
|
+
chunkBytes: z.ZodOptional<z.ZodNumber>;
|
|
54
|
+
}, z.core.$loose>>;
|
|
55
|
+
}, z.core.$loose>;
|
|
56
|
+
export type StorageHello = z.infer<typeof StorageHello>;
|
|
23
57
|
export {};
|
|
@@ -2,12 +2,19 @@ import { z } from "zod";
|
|
|
2
2
|
/** Client-side v1 wire contract only. No server/control/catalog filesystem imports.
|
|
3
3
|
* Keep matched to storage RPC; integration tests exercise the real authenticated parser.
|
|
4
4
|
*/
|
|
5
|
+
/** `chunkBytes` is this client's ceiling; the size actually used is negotiated
|
|
6
|
+
* down to what the storage authority's `hello` advertises (`limits.chunkBytes`),
|
|
7
|
+
* so a newer worker keeps talking to an older authority. A blob's chunks are
|
|
8
|
+
* appended in order, one round-trip each: with 64 KiB chunks a 3 MB bundle
|
|
9
|
+
* cost 32 round-trips through the relay — minutes per publication. */
|
|
5
10
|
export declare const STORAGE_LIMITS: {
|
|
6
11
|
readonly chunkBytes: number;
|
|
7
12
|
readonly blobBytes: number;
|
|
8
13
|
readonly requestBytes: number;
|
|
9
14
|
readonly treeEntries: 200000;
|
|
10
15
|
};
|
|
16
|
+
/** Chunk size assumed for an authority whose hello advertises no limits. */
|
|
17
|
+
export declare const LEGACY_CHUNK_BYTES: number;
|
|
11
18
|
export declare const StorageId: z.ZodString;
|
|
12
19
|
export declare const GitOid: z.ZodString;
|
|
13
20
|
export declare const BranchName: z.ZodString;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ricsam/r5d-worker",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.168",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/mjs/main.mjs",
|
|
6
6
|
"module": "./dist/mjs/main.mjs",
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"r5d-worker": "dist/mjs/main.mjs"
|
|
22
22
|
},
|
|
23
23
|
"dependencies": {
|
|
24
|
-
"@ricsam/r5d-api": "^0.0.
|
|
24
|
+
"@ricsam/r5d-api": "^0.0.168",
|
|
25
25
|
"node-pty": "1.1.0",
|
|
26
26
|
"zod": "^4.1.13",
|
|
27
27
|
"picomatch": "^4.0.3"
|