@exla/rudder 4.0.0-alpha.7 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -69
- package/dist/agent-efficiency/backend-observer.d.ts +13 -0
- package/dist/agent-efficiency/backend-observer.js +77 -0
- package/dist/agent-efficiency/backend-observer.js.map +1 -0
- package/dist/agent-efficiency/handoff.d.ts +9 -0
- package/dist/agent-efficiency/handoff.js +42 -0
- package/dist/agent-efficiency/handoff.js.map +1 -0
- package/dist/agent-efficiency/index.d.ts +11 -0
- package/dist/agent-efficiency/index.js +7 -0
- package/dist/agent-efficiency/index.js.map +1 -0
- package/dist/agent-efficiency/lessons.d.ts +55 -0
- package/dist/agent-efficiency/lessons.js +107 -0
- package/dist/agent-efficiency/lessons.js.map +1 -0
- package/dist/agent-efficiency/privacy.d.ts +6 -0
- package/dist/agent-efficiency/privacy.js +28 -0
- package/dist/agent-efficiency/privacy.js.map +1 -0
- package/dist/agent-efficiency/production.d.ts +16 -0
- package/dist/agent-efficiency/production.js +120 -0
- package/dist/agent-efficiency/production.js.map +1 -0
- package/dist/agent-efficiency/storage.d.ts +3 -0
- package/dist/agent-efficiency/storage.js +28 -0
- package/dist/agent-efficiency/storage.js.map +1 -0
- package/dist/agent-efficiency/telemetry.d.ts +59 -0
- package/dist/agent-efficiency/telemetry.js +143 -0
- package/dist/agent-efficiency/telemetry.js.map +1 -0
- package/dist/agent-efficiency/trace-lessons.d.ts +21 -0
- package/dist/agent-efficiency/trace-lessons.js +36 -0
- package/dist/agent-efficiency/trace-lessons.js.map +1 -0
- package/dist/agent-efficiency/waits.d.ts +52 -0
- package/dist/agent-efficiency/waits.js +131 -0
- package/dist/agent-efficiency/waits.js.map +1 -0
- package/dist/backends.js +14 -8
- package/dist/backends.js.map +1 -1
- package/dist/board/authority.d.ts +48 -0
- package/dist/board/authority.js +125 -0
- package/dist/board/authority.js.map +1 -0
- package/dist/board/board.js +6 -2
- package/dist/board/daemon.d.ts +3 -6
- package/dist/board/daemon.js +242 -222
- package/dist/board/daemon.js.map +1 -1
- package/dist/board/host-access.d.ts +8 -0
- package/dist/board/host-access.js +222 -0
- package/dist/board/host-access.js.map +1 -0
- package/dist/board/project-files.d.ts +12 -0
- package/dist/board/project-files.js +78 -0
- package/dist/board/project-files.js.map +1 -0
- package/dist/brain.d.ts +2 -1
- package/dist/brain.js +15 -21
- package/dist/brain.js.map +1 -1
- package/dist/claude-sdk-protocol.js +10 -7
- package/dist/claude-sdk-protocol.js.map +1 -1
- package/dist/cloud-build-profile.d.ts +22 -0
- package/dist/cloud-build-profile.js +160 -0
- package/dist/cloud-build-profile.js.map +1 -0
- package/dist/cloud.js +23 -470
- package/dist/cloud.js.map +1 -1
- package/dist/codex-binary.d.ts +1 -0
- package/dist/codex-binary.js +8 -1
- package/dist/codex-binary.js.map +1 -1
- package/dist/daemon.d.ts +2 -0
- package/dist/daemon.js +14 -2
- package/dist/daemon.js.map +1 -1
- package/dist/effort.d.ts +8 -0
- package/dist/effort.js +15 -2
- package/dist/effort.js.map +1 -1
- package/dist/engine-authority.js +16 -0
- package/dist/engine-authority.js.map +1 -1
- package/dist/engine.d.ts +1 -1
- package/dist/engine.js +11 -2
- package/dist/engine.js.map +1 -1
- package/dist/fabric-fetch.d.ts +25 -0
- package/dist/fabric-fetch.js +402 -0
- package/dist/fabric-fetch.js.map +1 -0
- package/dist/fabric-local.d.ts +5 -2
- package/dist/fabric-local.js +36 -1
- package/dist/fabric-local.js.map +1 -1
- package/dist/fabric-remote.d.ts +76 -1
- package/dist/fabric-remote.js +333 -61
- package/dist/fabric-remote.js.map +1 -1
- package/dist/git.js +23 -0
- package/dist/git.js.map +1 -1
- package/dist/graph.js +9 -1
- package/dist/graph.js.map +1 -1
- package/dist/jj.d.ts +3 -1
- package/dist/jj.js +34 -2
- package/dist/jj.js.map +1 -1
- package/dist/main.js +102 -310
- package/dist/main.js.map +1 -1
- package/dist/migration.d.ts +2 -0
- package/dist/migration.js +3 -0
- package/dist/migration.js.map +1 -1
- package/dist/models.js +11 -6
- package/dist/models.js.map +1 -1
- package/dist/native/darwin-arm64/native-manifest.json +19 -0
- package/dist/native/darwin-arm64/rudder-jj +0 -0
- package/dist/native/darwin-arm64/rudder-native +0 -0
- package/dist/native/darwin-x64/native-manifest.json +19 -0
- package/dist/native/darwin-x64/rudder-jj +0 -0
- package/dist/native/darwin-x64/rudder-native +0 -0
- package/dist/native/linux-x64/native-manifest.json +19 -0
- package/dist/native/linux-x64/rudder-jj +0 -0
- package/dist/native/linux-x64/rudder-native +0 -0
- package/dist/native-agents.js +3 -7
- package/dist/native-agents.js.map +1 -1
- package/dist/native-binary.js +11 -1
- package/dist/native-binary.js.map +1 -1
- package/dist/operations/cli.d.ts +23 -0
- package/dist/operations/cli.js +110 -0
- package/dist/operations/cli.js.map +1 -0
- package/dist/operations/registry.d.ts +1076 -0
- package/dist/operations/registry.js +69 -0
- package/dist/operations/registry.js.map +1 -0
- package/dist/operations/registry.json +124 -0
- package/dist/operations/requests.d.ts +167 -0
- package/dist/operations/requests.js +223 -0
- package/dist/operations/requests.js.map +1 -0
- package/dist/planning-ownership/index.d.ts +134 -0
- package/dist/planning-ownership/index.js +291 -0
- package/dist/planning-ownership/index.js.map +1 -0
- package/dist/planning-ownership/ingress.d.ts +30 -0
- package/dist/planning-ownership/ingress.js +41 -0
- package/dist/planning-ownership/ingress.js.map +1 -0
- package/dist/planning-ownership/store.d.ts +19 -0
- package/dist/planning-ownership/store.js +30 -0
- package/dist/planning-ownership/store.js.map +1 -0
- package/dist/private-access/cli.d.ts +22 -0
- package/dist/private-access/cli.js +76 -0
- package/dist/private-access/cli.js.map +1 -0
- package/dist/private-access/engine.d.ts +20 -0
- package/dist/private-access/engine.js +47 -0
- package/dist/private-access/engine.js.map +1 -0
- package/dist/private-access/home.d.ts +21 -0
- package/dist/private-access/home.js +80 -0
- package/dist/private-access/home.js.map +1 -0
- package/dist/private-access/previews.d.ts +101 -0
- package/dist/private-access/previews.js +447 -0
- package/dist/private-access/previews.js.map +1 -0
- package/dist/project-files/cli.d.ts +7 -0
- package/dist/project-files/cli.js +22 -0
- package/dist/project-files/cli.js.map +1 -0
- package/dist/project-files/engine.d.ts +14 -0
- package/dist/project-files/engine.js +52 -0
- package/dist/project-files/engine.js.map +1 -0
- package/dist/project-files/home.d.ts +13 -0
- package/dist/project-files/home.js +46 -0
- package/dist/project-files/home.js.map +1 -0
- package/dist/project-files/http.d.ts +9 -0
- package/dist/project-files/http.js +54 -0
- package/dist/project-files/http.js.map +1 -0
- package/dist/project-files/open.d.ts +10 -0
- package/dist/project-files/open.js +37 -0
- package/dist/project-files/open.js.map +1 -0
- package/dist/project-files/references.d.ts +9 -0
- package/dist/project-files/references.js +29 -0
- package/dist/project-files/references.js.map +1 -0
- package/dist/project-files/service.d.ts +32 -0
- package/dist/project-files/service.js +236 -0
- package/dist/project-files/service.js.map +1 -0
- package/dist/project-files/types.d.ts +40 -0
- package/dist/project-files/types.js +2 -0
- package/dist/project-files/types.js.map +1 -0
- package/dist/run-manager.d.ts +2 -0
- package/dist/run-manager.js +183 -13
- package/dist/run-manager.js.map +1 -1
- package/dist/run-timeline.d.ts +29 -0
- package/dist/run-timeline.js +112 -0
- package/dist/run-timeline.js.map +1 -0
- package/dist/scheduler.js +72 -13
- package/dist/scheduler.js.map +1 -1
- package/dist/surfaces.js +3 -8
- package/dist/surfaces.js.map +1 -1
- package/dist/tailscale-preview.d.ts +6 -2
- package/dist/tailscale-preview.js +58 -13
- package/dist/tailscale-preview.js.map +1 -1
- package/dist/task-routing.d.ts +2 -0
- package/dist/task-routing.js +42 -25
- package/dist/task-routing.js.map +1 -1
- package/dist/terminal-screen.d.ts +6 -0
- package/dist/terminal-screen.js +23 -0
- package/dist/terminal-screen.js.map +1 -0
- package/dist/tool-cache.d.ts +23 -0
- package/dist/tool-cache.js +144 -0
- package/dist/tool-cache.js.map +1 -0
- package/dist/types.d.ts +0 -17
- package/dist/types.js.map +1 -1
- package/dist/util.d.ts +8 -0
- package/dist/util.js +38 -2
- package/dist/util.js.map +1 -1
- package/dist/v4-migration/capture.d.ts +53 -0
- package/dist/v4-migration/capture.js +197 -0
- package/dist/v4-migration/capture.js.map +1 -0
- package/dist/v4-migration/cli.d.ts +4 -0
- package/dist/v4-migration/cli.js +38 -0
- package/dist/v4-migration/cli.js.map +1 -0
- package/dist/v4-migration/recovery.d.ts +43 -0
- package/dist/v4-migration/recovery.js +180 -0
- package/dist/v4-migration/recovery.js.map +1 -0
- package/dist/version-check.d.ts +2 -0
- package/dist/version-check.js +23 -0
- package/dist/version-check.js.map +1 -1
- package/dist/warm-snapshot.d.ts +55 -0
- package/dist/warm-snapshot.js +381 -0
- package/dist/warm-snapshot.js.map +1 -0
- package/package.json +4 -3
- package/scripts/copy-native.mjs +35 -48
- package/scripts/native-artifacts.mjs +248 -0
- package/dist/improve/advisor.d.ts +0 -10
- package/dist/improve/advisor.js +0 -115
- package/dist/improve/advisor.js.map +0 -1
- package/dist/improve/collect.d.ts +0 -49
- package/dist/improve/collect.js +0 -274
- package/dist/improve/collect.js.map +0 -1
- package/dist/improve/gate.d.ts +0 -17
- package/dist/improve/gate.js +0 -60
- package/dist/improve/gate.js.map +0 -1
- package/dist/improve/index.d.ts +0 -9
- package/dist/improve/index.js +0 -376
- package/dist/improve/index.js.map +0 -1
- package/dist/improve/judge.d.ts +0 -32
- package/dist/improve/judge.js +0 -114
- package/dist/improve/judge.js.map +0 -1
- package/dist/improve/mine.d.ts +0 -31
- package/dist/improve/mine.js +0 -192
- package/dist/improve/mine.js.map +0 -1
- package/dist/improve/propose.d.ts +0 -51
- package/dist/improve/propose.js +0 -150
- package/dist/improve/propose.js.map +0 -1
- package/dist/improve/rank.d.ts +0 -16
- package/dist/improve/rank.js +0 -49
- package/dist/improve/rank.js.map +0 -1
- package/dist/improve/schedule.d.ts +0 -1
- package/dist/improve/schedule.js +0 -95
- package/dist/improve/schedule.js.map +0 -1
- package/dist/improve/ship.d.ts +0 -25
- package/dist/improve/ship.js +0 -82
- package/dist/improve/ship.js.map +0 -1
- package/dist/improve/state.d.ts +0 -157
- package/dist/improve/state.js +0 -264
- package/dist/improve/state.js.map +0 -1
package/dist/fabric-remote.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type FabricObjectKind = "blob" | "directory" | "source_root" | "environment_root" | "checkpoint" | "action_key" | "action_result";
|
|
1
|
+
export type FabricObjectKind = "blob" | "file" | "directory" | "directory_shard" | "source_root" | "environment_root" | "checkpoint" | "runnable_root" | "action_key" | "action_result";
|
|
2
2
|
export interface FabricObjectRef {
|
|
3
3
|
kind: FabricObjectKind;
|
|
4
4
|
algorithm: "sha256";
|
|
@@ -25,6 +25,63 @@ export interface FabricTransferSource {
|
|
|
25
25
|
closure(root: FabricObjectRef): Promise<FabricObjectRef[]>;
|
|
26
26
|
read(reference: FabricObjectRef): Promise<Uint8Array>;
|
|
27
27
|
}
|
|
28
|
+
export interface FabricScope {
|
|
29
|
+
provider_id: string;
|
|
30
|
+
tenant_id: string;
|
|
31
|
+
}
|
|
32
|
+
export interface FabricPartialInventory {
|
|
33
|
+
format_version: 1;
|
|
34
|
+
owned_pin: string;
|
|
35
|
+
plan: FabricObjectRef;
|
|
36
|
+
directory: FabricObjectRef;
|
|
37
|
+
hardlinks: FabricObjectRef;
|
|
38
|
+
base: {
|
|
39
|
+
scope: FabricScope;
|
|
40
|
+
root_id: string;
|
|
41
|
+
root: FabricObjectRef;
|
|
42
|
+
};
|
|
43
|
+
objects: FabricObjectRef[];
|
|
44
|
+
}
|
|
45
|
+
export interface FabricPartialSource {
|
|
46
|
+
/** Keep the native receipt opaque: its generations may exceed JS integers. */
|
|
47
|
+
pinPartial(receiptJson: string): Promise<FabricPartialInventory>;
|
|
48
|
+
unpin(ownedPinId: string): Promise<void>;
|
|
49
|
+
read(reference: FabricObjectRef): Promise<Uint8Array>;
|
|
50
|
+
}
|
|
51
|
+
/** This inventory retains a checkpoint DATA bundle. Its ActionResult root is
|
|
52
|
+
* not a RunnableRoot receipt, a complete local pin or accepted source history. */
|
|
53
|
+
export interface FabricPartialCheckpointInventory {
|
|
54
|
+
format_version: 1;
|
|
55
|
+
owned_pin: string;
|
|
56
|
+
publication_id: string;
|
|
57
|
+
publication: FabricObjectRef;
|
|
58
|
+
q: FabricObjectRef;
|
|
59
|
+
source: FabricObjectRef;
|
|
60
|
+
runnable: FabricObjectRef;
|
|
61
|
+
plan: FabricObjectRef;
|
|
62
|
+
freeze_intent: FabricObjectRef;
|
|
63
|
+
drain_record: FabricObjectRef;
|
|
64
|
+
base: FabricPartialInventory["base"];
|
|
65
|
+
objects: FabricObjectRef[];
|
|
66
|
+
}
|
|
67
|
+
export interface FabricPartialCheckpointSource {
|
|
68
|
+
pinPartialCheckpoint(receiptJson: string): Promise<FabricPartialCheckpointInventory>;
|
|
69
|
+
unpin(ownedPinId: string): Promise<void>;
|
|
70
|
+
read(reference: FabricObjectRef): Promise<Uint8Array>;
|
|
71
|
+
}
|
|
72
|
+
export interface FabricPartialCheckpointUploadResult {
|
|
73
|
+
format_version: 1;
|
|
74
|
+
kind: "partial_checkpoint_data";
|
|
75
|
+
source_role: "quiesced_source_complement_data";
|
|
76
|
+
publication_id: string;
|
|
77
|
+
publication: FabricObjectRef;
|
|
78
|
+
runnable: FabricObjectRef;
|
|
79
|
+
q: FabricObjectRef;
|
|
80
|
+
bundle: FabricRootReceipt;
|
|
81
|
+
uploaded_objects: number;
|
|
82
|
+
uploaded_bytes: number;
|
|
83
|
+
cleanup_warning?: string;
|
|
84
|
+
}
|
|
28
85
|
export interface FabricTransferResult {
|
|
29
86
|
receipt: FabricRootReceipt;
|
|
30
87
|
uploaded_objects: number;
|
|
@@ -58,6 +115,8 @@ export declare class FabricRemoteError extends Error {
|
|
|
58
115
|
readonly status?: number | undefined;
|
|
59
116
|
constructor(code: string, message: string, status?: number | undefined);
|
|
60
117
|
}
|
|
118
|
+
export declare function validateFabricPartialInventory(value: unknown): FabricPartialInventory;
|
|
119
|
+
export declare function validateFabricPartialCheckpointInventory(value: unknown): FabricPartialCheckpointInventory;
|
|
61
120
|
export declare function validateFabricReference(value: unknown): FabricObjectRef;
|
|
62
121
|
export declare function verifyFabricObject(reference: FabricObjectRef, bytes: Uint8Array): void;
|
|
63
122
|
export declare class FabricRemoteClient {
|
|
@@ -67,6 +126,8 @@ export declare class FabricRemoteClient {
|
|
|
67
126
|
private readonly requestTimeoutMs;
|
|
68
127
|
private readonly leaseTtlMs;
|
|
69
128
|
private readonly buffered;
|
|
129
|
+
private inFlight;
|
|
130
|
+
private readonly scope;
|
|
70
131
|
constructor(options: {
|
|
71
132
|
/** Full authenticated endpoint base, ending /api/rudder/fabric. */
|
|
72
133
|
baseUrl: string;
|
|
@@ -75,16 +136,30 @@ export declare class FabricRemoteClient {
|
|
|
75
136
|
concurrency?: number;
|
|
76
137
|
requestTimeoutMs?: number;
|
|
77
138
|
leaseTtlMs?: number;
|
|
139
|
+
/** Trusted host configuration, never supplied by a saved receipt. */
|
|
140
|
+
scope?: FabricScope;
|
|
78
141
|
});
|
|
79
142
|
private request;
|
|
80
143
|
private json;
|
|
81
144
|
beginStaging(): Promise<FabricLease>;
|
|
82
145
|
renewStaging(leaseId: string): Promise<FabricLease>;
|
|
146
|
+
abandonStaging(leaseId: string): Promise<void>;
|
|
83
147
|
missing(objects: FabricObjectRef[]): Promise<FabricObjectRef[]>;
|
|
84
148
|
putObject(leaseId: string, reference: FabricObjectRef, bytes: Uint8Array): Promise<void>;
|
|
149
|
+
private putRetainedObject;
|
|
85
150
|
getObject(reference: FabricObjectRef): Promise<Buffer>;
|
|
151
|
+
private getRetainedObject;
|
|
86
152
|
publishRoot(root: string, leaseId: string, object: FabricObjectRef): Promise<FabricRootReceipt>;
|
|
87
153
|
getRoot(root: string): Promise<FabricRootReceipt>;
|
|
154
|
+
private uploadInventory;
|
|
155
|
+
/** Publishes the complete resulting directory in Cloud using only retained
|
|
156
|
+
* new objects. This is data replication, not accepted source/quiescence or a
|
|
157
|
+
* newly runnable checkpoint. The host must publish those separately. */
|
|
158
|
+
uploadPartialTree(root: string, receiptJson: string, source: FabricPartialSource): Promise<FabricTransferResult>;
|
|
159
|
+
/** Clone the producer's exact retained DATA and publish its whole proof
|
|
160
|
+
* closure. Repeated calls first reconcile the same immutable publication ID;
|
|
161
|
+
* a transport failure never automatically retries an uncertain POST. */
|
|
162
|
+
uploadPartialCheckpoint(publicationId: string, receiptJson: string, source: FabricPartialCheckpointSource): Promise<FabricPartialCheckpointUploadResult>;
|
|
88
163
|
uploadRoot(root: string, object: FabricObjectRef, source: FabricTransferSource): Promise<FabricTransferResult>;
|
|
89
164
|
downloadRoot(root: string, target: FabricTransferTarget, pin: string): Promise<FabricDownloadResult>;
|
|
90
165
|
}
|
package/dist/fabric-remote.js
CHANGED
|
@@ -14,14 +14,84 @@ export class FabricRemoteError extends Error {
|
|
|
14
14
|
}
|
|
15
15
|
}
|
|
16
16
|
const fail = (code, message) => { throw new FabricRemoteError(code, message); };
|
|
17
|
-
const kinds = new Set(["blob", "directory", "source_root", "environment_root", "checkpoint", "action_key", "action_result"]);
|
|
17
|
+
const kinds = new Set(["blob", "file", "directory", "directory_shard", "source_root", "environment_root", "checkpoint", "runnable_root", "action_key", "action_result"]);
|
|
18
18
|
const identity = (object) => `${object.kind}:${object.digest}:${object.size_bytes}`;
|
|
19
|
+
function scope(value) {
|
|
20
|
+
const item = value;
|
|
21
|
+
if (!item || Object.keys(item).length !== 2 || [item.provider_id, item.tenant_id].some((part) => typeof part !== "string" || !/^[A-Za-z0-9._:-]{1,128}$/.test(part)))
|
|
22
|
+
fail("invalid_scope", "Invalid provider and tenant binding");
|
|
23
|
+
return { provider_id: item.provider_id, tenant_id: item.tenant_id };
|
|
24
|
+
}
|
|
25
|
+
export function validateFabricPartialInventory(value) {
|
|
26
|
+
const item = value;
|
|
27
|
+
if (!item || Object.keys(item).length !== 7 || item.format_version !== 1 || typeof item.owned_pin !== "string" || !/^transfer:partial:rpc:[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/.test(item.owned_pin) || !item.base || Object.keys(item.base).length !== 3 || !Array.isArray(item.objects) || item.objects.length < 1 || item.objects.length > 16_384)
|
|
28
|
+
fail("invalid_source", "Invalid partial source inventory");
|
|
29
|
+
const plan = validateFabricReference(item.plan), directory = validateFabricReference(item.directory), hardlinks = validateFabricReference(item.hardlinks);
|
|
30
|
+
const base = { scope: scope(item.base.scope), root_id: rootId(item.base.root_id), root: validateFabricReference(item.base.root) };
|
|
31
|
+
if (plan.kind !== "blob" || directory.kind !== "directory" || hardlinks.kind !== "blob" || base.root.kind !== "runnable_root")
|
|
32
|
+
fail("invalid_source", "Invalid partial source object kinds");
|
|
33
|
+
const objects = item.objects.map(validateFabricReference);
|
|
34
|
+
const seen = new Set();
|
|
35
|
+
let total = 0;
|
|
36
|
+
for (const object of objects) {
|
|
37
|
+
const key = identity(object);
|
|
38
|
+
if (seen.has(key))
|
|
39
|
+
fail("invalid_source", "Partial inventory repeats an object");
|
|
40
|
+
seen.add(key);
|
|
41
|
+
total += object.size_bytes;
|
|
42
|
+
}
|
|
43
|
+
if (!seen.has(identity(plan)) || total > 4 * 1024 ** 3)
|
|
44
|
+
fail("invalid_source", "Partial inventory is incomplete or exceeds byte limit");
|
|
45
|
+
return { format_version: 1, owned_pin: item.owned_pin, plan, directory, hardlinks, base, objects };
|
|
46
|
+
}
|
|
47
|
+
const partialCheckpointPin = /^transfer:partial-checkpoint:rpc:[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/;
|
|
48
|
+
function partialCheckpointId(value) {
|
|
49
|
+
if (typeof value !== "string" || !/^pcd1-[a-f0-9]{64}$/.test(value))
|
|
50
|
+
fail("invalid_source", "Invalid checkpoint data publication identity");
|
|
51
|
+
return value;
|
|
52
|
+
}
|
|
53
|
+
export function validateFabricPartialCheckpointInventory(value) {
|
|
54
|
+
const keys = ["format_version", "owned_pin", "publication_id", "publication", "q", "source", "runnable", "plan", "freeze_intent", "drain_record", "base", "objects"];
|
|
55
|
+
const item = value;
|
|
56
|
+
if (!item || typeof item !== "object" || Array.isArray(item) || Object.keys(item).length !== keys.length
|
|
57
|
+
|| keys.some((key) => !Object.hasOwn(item, key)) || item.format_version !== 1
|
|
58
|
+
|| typeof item.owned_pin !== "string" || !partialCheckpointPin.test(item.owned_pin)
|
|
59
|
+
|| !item.base || typeof item.base !== "object" || Array.isArray(item.base) || Object.keys(item.base).length !== 3
|
|
60
|
+
|| ["scope", "root_id", "root"].some((key) => !Object.hasOwn(item.base, key))
|
|
61
|
+
|| !Array.isArray(item.objects) || item.objects.length < 1 || item.objects.length > 16_384)
|
|
62
|
+
fail("invalid_source", "Invalid checkpoint data transfer inventory");
|
|
63
|
+
const publication_id = partialCheckpointId(item.publication_id);
|
|
64
|
+
const publication = validateFabricReference(item.publication), q = validateFabricReference(item.q);
|
|
65
|
+
const source = validateFabricReference(item.source), runnable = validateFabricReference(item.runnable);
|
|
66
|
+
const plan = validateFabricReference(item.plan), freeze_intent = validateFabricReference(item.freeze_intent), drain_record = validateFabricReference(item.drain_record);
|
|
67
|
+
const base = { scope: scope(item.base.scope), root_id: rootId(item.base.root_id), root: validateFabricReference(item.base.root) };
|
|
68
|
+
if (publication.kind !== "action_result" || publication.size_bytes > 64 * 1024 || source.kind !== "source_root"
|
|
69
|
+
|| runnable.kind !== "runnable_root" || base.root.kind !== "runnable_root"
|
|
70
|
+
|| q.kind !== "blob" || q.size_bytes > 32 * 1024 || plan.kind !== "blob" || plan.size_bytes > 4 * 1024 * 1024
|
|
71
|
+
|| freeze_intent.kind !== "blob" || freeze_intent.size_bytes > 64 * 1024
|
|
72
|
+
|| drain_record.kind !== "blob" || drain_record.size_bytes > 32 * 1024)
|
|
73
|
+
fail("invalid_source", "Invalid checkpoint data object kinds or bounds");
|
|
74
|
+
const objects = item.objects.map(validateFabricReference), seen = new Set();
|
|
75
|
+
let total = 0;
|
|
76
|
+
for (const object of objects) {
|
|
77
|
+
const key = identity(object);
|
|
78
|
+
if (seen.has(key))
|
|
79
|
+
fail("invalid_source", "Checkpoint data inventory repeats an object");
|
|
80
|
+
seen.add(key);
|
|
81
|
+
total += object.size_bytes;
|
|
82
|
+
}
|
|
83
|
+
if (total > 4 * 1024 ** 3 || [publication, q, source, runnable, plan, freeze_intent, drain_record].some((object) => !seen.has(identity(object))))
|
|
84
|
+
fail("invalid_source", "Checkpoint data inventory omits retained proof or exceeds its byte limit");
|
|
85
|
+
return { format_version: 1, owned_pin: item.owned_pin, publication_id, publication, q, source, runnable, plan, freeze_intent, drain_record, base, objects };
|
|
86
|
+
}
|
|
19
87
|
export function validateFabricReference(value) {
|
|
20
88
|
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
21
89
|
fail("invalid_reference", "Invalid object reference");
|
|
22
90
|
const object = value;
|
|
23
91
|
if (Object.keys(object).length !== 4 || typeof object.kind !== "string" || !kinds.has(object.kind) || object.algorithm !== "sha256" || typeof object.digest !== "string" || !/^[a-f0-9]{64}$/.test(object.digest) || !Number.isSafeInteger(object.size_bytes) || Number(object.size_bytes) < 0 || Number(object.size_bytes) > 256 * 1024 * 1024 || (object.kind !== "blob" && Number(object.size_bytes) > 4 * 1024 * 1024))
|
|
24
92
|
fail("invalid_reference", "Invalid typed SHA-256 object reference");
|
|
93
|
+
if ((object.kind === "file" || object.kind === "directory_shard" || object.kind === "runnable_root") && Number(object.size_bytes) > 64 * 1024)
|
|
94
|
+
fail("invalid_reference", "Content descriptor exceeds its format byte limit");
|
|
25
95
|
return { kind: object.kind, algorithm: "sha256", digest: object.digest, size_bytes: object.size_bytes };
|
|
26
96
|
}
|
|
27
97
|
export function verifyFabricObject(reference, bytes) {
|
|
@@ -46,24 +116,54 @@ function receipt(value, expectedId, expectedRoot) {
|
|
|
46
116
|
fail("invalid_response", "Publication receipt refers to another root");
|
|
47
117
|
return { ...item, root };
|
|
48
118
|
}
|
|
49
|
-
/** A single client cannot hold several large objects at once, even when the
|
|
50
|
-
* caller raises small-object concurrency. FIFO admission prevents starvation. */
|
|
51
119
|
class ByteBudget {
|
|
52
120
|
available = 256 * 1024 * 1024;
|
|
53
121
|
queue = [];
|
|
54
|
-
acquire(bytes) {
|
|
55
|
-
|
|
122
|
+
acquire(bytes, timeoutMs, signal) {
|
|
123
|
+
if (signal.aborted)
|
|
124
|
+
return Promise.reject(new FabricRemoteError("transfer_cancelled", "Transfer batch has stopped"));
|
|
125
|
+
if (this.queue.length >= 32)
|
|
126
|
+
return Promise.reject(new FabricRemoteError("buffer_busy", "Transfer buffer admission is busy"));
|
|
127
|
+
return new Promise((resolve, reject) => {
|
|
128
|
+
const cancel = (error) => {
|
|
129
|
+
const index = this.queue.indexOf(item);
|
|
130
|
+
if (index < 0)
|
|
131
|
+
return;
|
|
132
|
+
this.queue.splice(index, 1);
|
|
133
|
+
item.cleanup();
|
|
134
|
+
reject(error);
|
|
135
|
+
this.drain();
|
|
136
|
+
};
|
|
137
|
+
const aborted = () => cancel(new FabricRemoteError("transfer_cancelled", "Transfer batch has stopped"));
|
|
138
|
+
const timer = setTimeout(() => cancel(new FabricRemoteError("request_timeout", "Transfer buffer admission exceeded its deadline")), timeoutMs);
|
|
139
|
+
const item = { bytes: Math.max(1, bytes), resolve, cleanup: () => { clearTimeout(timer); signal.removeEventListener("abort", aborted); } };
|
|
140
|
+
signal.addEventListener("abort", aborted, { once: true });
|
|
141
|
+
this.queue.push(item);
|
|
142
|
+
this.drain();
|
|
143
|
+
});
|
|
56
144
|
}
|
|
57
145
|
drain() {
|
|
58
146
|
while (this.queue.length && this.queue[0].bytes <= this.available) {
|
|
59
147
|
const item = this.queue.shift();
|
|
148
|
+
item.cleanup();
|
|
60
149
|
this.available -= item.bytes;
|
|
61
|
-
let
|
|
62
|
-
|
|
63
|
-
released =
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
150
|
+
let owners = 1;
|
|
151
|
+
const releaseOwner = () => {
|
|
152
|
+
let released = false;
|
|
153
|
+
return () => { if (!released) {
|
|
154
|
+
released = true;
|
|
155
|
+
if (--owners === 0) {
|
|
156
|
+
this.available += item.bytes;
|
|
157
|
+
this.drain();
|
|
158
|
+
}
|
|
159
|
+
} };
|
|
160
|
+
};
|
|
161
|
+
item.resolve({ release: releaseOwner(), retain: () => {
|
|
162
|
+
if (!owners)
|
|
163
|
+
fail("invalid_source", "Transfer buffer was already released");
|
|
164
|
+
owners++;
|
|
165
|
+
return releaseOwner();
|
|
166
|
+
} });
|
|
67
167
|
}
|
|
68
168
|
}
|
|
69
169
|
}
|
|
@@ -74,6 +174,8 @@ export class FabricRemoteClient {
|
|
|
74
174
|
requestTimeoutMs;
|
|
75
175
|
leaseTtlMs;
|
|
76
176
|
buffered = new ByteBudget();
|
|
177
|
+
inFlight = 0;
|
|
178
|
+
scope;
|
|
77
179
|
constructor(options) {
|
|
78
180
|
this.options = options;
|
|
79
181
|
this.base = new URL(options.baseUrl.endsWith("/") ? options.baseUrl : `${options.baseUrl}/`);
|
|
@@ -82,10 +184,19 @@ export class FabricRemoteClient {
|
|
|
82
184
|
this.concurrency = options.concurrency ?? 1;
|
|
83
185
|
this.requestTimeoutMs = options.requestTimeoutMs ?? 120_000;
|
|
84
186
|
this.leaseTtlMs = options.leaseTtlMs ?? 300_000;
|
|
187
|
+
this.scope = options.scope === undefined ? undefined : scope(options.scope);
|
|
85
188
|
if (!Number.isInteger(this.concurrency) || this.concurrency < 1 || this.concurrency > 8 || !Number.isInteger(this.requestTimeoutMs) || this.requestTimeoutMs < 1 || this.requestTimeoutMs > 120_000 || !Number.isInteger(this.leaseTtlMs) || this.leaseTtlMs < 1000 || this.leaseTtlMs > 3_600_000)
|
|
86
189
|
fail("invalid_limits", "Invalid transfer concurrency or deadline");
|
|
87
190
|
}
|
|
88
|
-
async request(path, method, body, binary, headers = {}, maxResponseBytes = 2 * 1024 * 1024) {
|
|
191
|
+
async request(path, method, body, binary, headers = {}, maxResponseBytes = 2 * 1024 * 1024, releaseBody) {
|
|
192
|
+
// A deadline bounds the caller's wait, not an abort-ignoring transport's
|
|
193
|
+
// lifetime. Keep both its request slot and any payload reservation until
|
|
194
|
+
// authentication, fetch and response consumption have actually settled.
|
|
195
|
+
if (this.inFlight >= this.concurrency) {
|
|
196
|
+
releaseBody?.();
|
|
197
|
+
fail("request_busy", "Fabric request admission is busy");
|
|
198
|
+
}
|
|
199
|
+
this.inFlight++;
|
|
89
200
|
const controller = new AbortController();
|
|
90
201
|
let timer;
|
|
91
202
|
const work = (async () => {
|
|
@@ -134,7 +245,7 @@ export class FabricRemoteClient {
|
|
|
134
245
|
await reader.cancel().catch(() => { });
|
|
135
246
|
}
|
|
136
247
|
return destination ? destination.subarray(0, bytes) : Buffer.concat(chunks);
|
|
137
|
-
})();
|
|
248
|
+
})().finally(() => { this.inFlight--; releaseBody?.(); });
|
|
138
249
|
try {
|
|
139
250
|
return await Promise.race([work, new Promise((_, reject) => { timer = setTimeout(() => { controller.abort(); reject(new FabricRemoteError("request_timeout", "Fabric request exceeded its deadline")); }, this.requestTimeoutMs); })]);
|
|
140
251
|
}
|
|
@@ -162,6 +273,12 @@ export class FabricRemoteClient {
|
|
|
162
273
|
lease({ format_version: 1, lease_id: leaseId, expires_at_ms: 1 });
|
|
163
274
|
return lease(await this.json(`staging/${leaseId}/renew`, "POST", { format_version: 1, ttl_ms: this.leaseTtlMs }));
|
|
164
275
|
}
|
|
276
|
+
async abandonStaging(leaseId) {
|
|
277
|
+
lease({ format_version: 1, lease_id: leaseId, expires_at_ms: 1 });
|
|
278
|
+
const response = await this.json(`staging/${leaseId}`, "DELETE", { format_version: 1 });
|
|
279
|
+
if (!response || Object.keys(response).length !== 1 || response.format_version !== 1)
|
|
280
|
+
fail("invalid_response", "Invalid staging cleanup acknowledgment");
|
|
281
|
+
}
|
|
165
282
|
async missing(objects) {
|
|
166
283
|
if (objects.length > 256)
|
|
167
284
|
fail("too_large", "Missing-object request exceeds batch limit");
|
|
@@ -175,13 +292,19 @@ export class FabricRemoteClient {
|
|
|
175
292
|
fail("invalid_response", "Server requested an undeclared or duplicate object"); seen.add(key); return reference; });
|
|
176
293
|
}
|
|
177
294
|
async putObject(leaseId, reference, bytes) {
|
|
295
|
+
await this.putRetainedObject(leaseId, reference, bytes);
|
|
296
|
+
}
|
|
297
|
+
async putRetainedObject(leaseId, reference, bytes, reservation) {
|
|
178
298
|
verifyFabricObject(reference, bytes);
|
|
179
299
|
lease({ format_version: 1, lease_id: leaseId, expires_at_ms: 1 });
|
|
180
|
-
await this.request(`objects/${reference.kind}/${reference.digest}/${reference.size_bytes}`, "PUT", undefined, bytes, { "X-Rudder-Staging-Lease": leaseId });
|
|
300
|
+
await this.request(`objects/${reference.kind}/${reference.digest}/${reference.size_bytes}`, "PUT", undefined, bytes, { "X-Rudder-Staging-Lease": leaseId }, 2 * 1024 * 1024, reservation?.retain());
|
|
181
301
|
}
|
|
182
302
|
async getObject(reference) {
|
|
303
|
+
return this.getRetainedObject(reference);
|
|
304
|
+
}
|
|
305
|
+
async getRetainedObject(reference, reservation) {
|
|
183
306
|
validateFabricReference(reference);
|
|
184
|
-
const bytes = await this.request(`objects/${reference.kind}/${reference.digest}/${reference.size_bytes}`, "GET", undefined, undefined, {}, reference.size_bytes);
|
|
307
|
+
const bytes = await this.request(`objects/${reference.kind}/${reference.digest}/${reference.size_bytes}`, "GET", undefined, undefined, {}, reference.size_bytes, reservation?.retain());
|
|
185
308
|
verifyFabricObject(reference, bytes);
|
|
186
309
|
return bytes;
|
|
187
310
|
}
|
|
@@ -195,70 +318,54 @@ export class FabricRemoteClient {
|
|
|
195
318
|
rootId(root);
|
|
196
319
|
return receipt(await this.json(`roots/${encodeURIComponent(root)}`, "GET"), root);
|
|
197
320
|
}
|
|
198
|
-
async
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
let ownedPin;
|
|
202
|
-
try {
|
|
203
|
-
ownedPin = await source.pin(object);
|
|
204
|
-
}
|
|
205
|
-
catch {
|
|
206
|
-
return fail("source_failed", "Could not pin the local transfer source");
|
|
207
|
-
}
|
|
208
|
-
if (typeof ownedPin !== "string" || !ownedPin.trim())
|
|
209
|
-
fail("invalid_source", "Source did not return an owned retention pin");
|
|
321
|
+
async uploadInventory(root, object, all, source) {
|
|
322
|
+
let staging = await this.beginStaging();
|
|
323
|
+
let renewal;
|
|
210
324
|
let result;
|
|
211
|
-
|
|
325
|
+
const ensureLease = async (force = false) => {
|
|
326
|
+
if (!force && staging.expires_at_ms > Date.now() + Math.max(70_000, this.leaseTtlMs / 3))
|
|
327
|
+
return;
|
|
328
|
+
renewal ??= this.renewStaging(staging.lease_id).then((updated) => { if (updated.lease_id !== staging.lease_id)
|
|
329
|
+
fail("invalid_response", "Lease renewal changed identity"); staging = updated; }).finally(() => { renewal = undefined; });
|
|
330
|
+
await renewal;
|
|
331
|
+
};
|
|
212
332
|
try {
|
|
213
|
-
const inventory = await source.closure(object);
|
|
214
|
-
if (!Array.isArray(inventory) || inventory.length > 100_000)
|
|
215
|
-
fail("too_large", "Source inventory exceeds object limit");
|
|
216
|
-
const unique = new Map();
|
|
217
|
-
let total = 0;
|
|
218
|
-
for (const value of inventory) {
|
|
219
|
-
const reference = validateFabricReference(value);
|
|
220
|
-
if (!unique.has(identity(reference))) {
|
|
221
|
-
unique.set(identity(reference), reference);
|
|
222
|
-
total += reference.size_bytes;
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
|
-
if (!unique.has(identity(object)) || total > 4 * 1024 ** 3)
|
|
226
|
-
fail("invalid_source", "Source inventory is incomplete or exceeds byte limit");
|
|
227
|
-
let staging = await this.beginStaging();
|
|
228
|
-
let renewal;
|
|
229
|
-
const ensureLease = async (force = false) => {
|
|
230
|
-
if (!force && staging.expires_at_ms > Date.now() + Math.max(70_000, this.leaseTtlMs / 3))
|
|
231
|
-
return;
|
|
232
|
-
renewal ??= this.renewStaging(staging.lease_id).then((updated) => { if (updated.lease_id !== staging.lease_id)
|
|
233
|
-
fail("invalid_response", "Lease renewal changed identity"); staging = updated; }).finally(() => { renewal = undefined; });
|
|
234
|
-
await renewal;
|
|
235
|
-
};
|
|
236
333
|
let uploaded = 0, uploadedBytes = 0;
|
|
237
|
-
const all = [...unique.values()];
|
|
238
334
|
for (let start = 0; start < all.length; start += 256) {
|
|
239
335
|
await ensureLease();
|
|
240
336
|
const missing = await this.missing(all.slice(start, start + 256));
|
|
241
337
|
let cursor = 0;
|
|
242
338
|
let workerFailure;
|
|
243
|
-
|
|
339
|
+
const cancelled = new AbortController();
|
|
340
|
+
// The local retention stays held until every reader has settled.
|
|
244
341
|
await Promise.all(Array.from({ length: Math.min(this.concurrency, missing.length) }, async () => {
|
|
245
342
|
while (cursor < missing.length && !workerFailure) {
|
|
246
343
|
const reference = missing[cursor++];
|
|
247
|
-
|
|
344
|
+
let reservation;
|
|
248
345
|
try {
|
|
346
|
+
reservation = await this.buffered.acquire(reference.size_bytes, this.requestTimeoutMs, cancelled.signal);
|
|
347
|
+
if (workerFailure)
|
|
348
|
+
continue;
|
|
249
349
|
await ensureLease();
|
|
350
|
+
if (workerFailure)
|
|
351
|
+
continue;
|
|
250
352
|
const bytes = await source.read(reference);
|
|
353
|
+
if (workerFailure)
|
|
354
|
+
continue;
|
|
251
355
|
verifyFabricObject(reference, bytes);
|
|
252
356
|
await ensureLease();
|
|
253
|
-
|
|
357
|
+
if (workerFailure)
|
|
358
|
+
continue;
|
|
359
|
+
await this.putRetainedObject(staging.lease_id, reference, bytes, reservation);
|
|
254
360
|
uploaded++;
|
|
255
361
|
uploadedBytes += bytes.byteLength;
|
|
256
362
|
}
|
|
257
363
|
catch (error) {
|
|
258
|
-
workerFailure
|
|
364
|
+
workerFailure ||= error || new FabricRemoteError("source_failed", "Pinned source read failed");
|
|
365
|
+
cancelled.abort();
|
|
259
366
|
}
|
|
260
367
|
finally {
|
|
261
|
-
release();
|
|
368
|
+
reservation?.release();
|
|
262
369
|
}
|
|
263
370
|
}
|
|
264
371
|
}));
|
|
@@ -267,6 +374,167 @@ export class FabricRemoteClient {
|
|
|
267
374
|
}
|
|
268
375
|
await ensureLease(true);
|
|
269
376
|
result = { receipt: await this.publishRoot(root, staging.lease_id, object), uploaded_objects: uploaded, uploaded_bytes: uploadedBytes };
|
|
377
|
+
return result;
|
|
378
|
+
}
|
|
379
|
+
finally {
|
|
380
|
+
// Publication also removes its lease; DELETE is idempotent after commit.
|
|
381
|
+
// Failure keeps at most the bounded lease lifetime, never a success claim.
|
|
382
|
+
try {
|
|
383
|
+
await this.abandonStaging(staging.lease_id);
|
|
384
|
+
}
|
|
385
|
+
catch {
|
|
386
|
+
if (result)
|
|
387
|
+
result.cleanup_warning = "Remote publication committed; staging cleanup was not confirmed";
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
/** Publishes the complete resulting directory in Cloud using only retained
|
|
392
|
+
* new objects. This is data replication, not accepted source/quiescence or a
|
|
393
|
+
* newly runnable checkpoint. The host must publish those separately. */
|
|
394
|
+
async uploadPartialTree(root, receiptJson, source) {
|
|
395
|
+
rootId(root);
|
|
396
|
+
if (!this.scope)
|
|
397
|
+
fail("scope_required", "Partial transfer requires a trusted provider and tenant binding");
|
|
398
|
+
if (typeof receiptJson !== "string" || Buffer.byteLength(receiptJson) > 64 * 1024)
|
|
399
|
+
fail("invalid_source", "Invalid opaque partial receipt");
|
|
400
|
+
let pinned;
|
|
401
|
+
try {
|
|
402
|
+
pinned = await source.pinPartial(receiptJson);
|
|
403
|
+
}
|
|
404
|
+
catch (error) {
|
|
405
|
+
throw error instanceof FabricRemoteError ? error : new FabricRemoteError("source_failed", "Could not retain the partial transfer source");
|
|
406
|
+
}
|
|
407
|
+
const issuedPin = pinned?.owned_pin;
|
|
408
|
+
if (typeof issuedPin !== "string" || !/^transfer:partial:rpc:[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/.test(issuedPin))
|
|
409
|
+
fail("invalid_source", "Partial source did not return a private retention identity");
|
|
410
|
+
let result;
|
|
411
|
+
let failure;
|
|
412
|
+
try {
|
|
413
|
+
const inventory = validateFabricPartialInventory(pinned);
|
|
414
|
+
if (inventory.owned_pin !== issuedPin)
|
|
415
|
+
fail("invalid_source", "Partial source changed its issued retention identity");
|
|
416
|
+
if (inventory.base.scope.provider_id !== this.scope.provider_id || inventory.base.scope.tenant_id !== this.scope.tenant_id)
|
|
417
|
+
fail("scope_mismatch", "Partial source belongs to another provider or tenant");
|
|
418
|
+
const base = await this.getRoot(inventory.base.root_id);
|
|
419
|
+
if (identity(base.root) !== identity(inventory.base.root))
|
|
420
|
+
fail("base_mismatch", "Partial source base no longer matches its immutable Cloud publication");
|
|
421
|
+
result = await this.uploadInventory(root, inventory.directory, inventory.objects, source);
|
|
422
|
+
}
|
|
423
|
+
catch (error) {
|
|
424
|
+
failure = error instanceof FabricRemoteError ? error : new FabricRemoteError("source_failed", "Partial source transfer failed");
|
|
425
|
+
}
|
|
426
|
+
finally {
|
|
427
|
+
try {
|
|
428
|
+
await source.unpin(issuedPin);
|
|
429
|
+
}
|
|
430
|
+
catch {
|
|
431
|
+
if (result)
|
|
432
|
+
result.cleanup_warning = "Remote publication committed; its temporary local retention needs cleanup";
|
|
433
|
+
else
|
|
434
|
+
failure ??= new FabricRemoteError("source_cleanup_failed", "Temporary local retention needs cleanup");
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
if (failure)
|
|
438
|
+
throw failure;
|
|
439
|
+
return result;
|
|
440
|
+
}
|
|
441
|
+
/** Clone the producer's exact retained DATA and publish its whole proof
|
|
442
|
+
* closure. Repeated calls first reconcile the same immutable publication ID;
|
|
443
|
+
* a transport failure never automatically retries an uncertain POST. */
|
|
444
|
+
async uploadPartialCheckpoint(publicationId, receiptJson, source) {
|
|
445
|
+
partialCheckpointId(publicationId);
|
|
446
|
+
const trustedScope = this.scope;
|
|
447
|
+
if (!trustedScope)
|
|
448
|
+
return fail("scope_required", "Checkpoint transfer requires a trusted provider and tenant binding");
|
|
449
|
+
if (typeof receiptJson !== "string" || Buffer.byteLength(receiptJson) > 64 * 1024)
|
|
450
|
+
fail("invalid_source", "Invalid opaque checkpoint data receipt");
|
|
451
|
+
let pinned;
|
|
452
|
+
try {
|
|
453
|
+
pinned = await source.pinPartialCheckpoint(receiptJson);
|
|
454
|
+
}
|
|
455
|
+
catch (error) {
|
|
456
|
+
throw error instanceof FabricRemoteError ? error : new FabricRemoteError("source_failed", "Could not retain the checkpoint data transfer source");
|
|
457
|
+
}
|
|
458
|
+
const issuedPin = pinned?.owned_pin;
|
|
459
|
+
if (typeof issuedPin !== "string" || !partialCheckpointPin.test(issuedPin))
|
|
460
|
+
fail("invalid_source", "Checkpoint source did not return a private retention identity");
|
|
461
|
+
let result;
|
|
462
|
+
let failure;
|
|
463
|
+
try {
|
|
464
|
+
const inventory = validateFabricPartialCheckpointInventory(pinned);
|
|
465
|
+
if (inventory.owned_pin !== issuedPin)
|
|
466
|
+
fail("invalid_source", "Checkpoint source changed its issued retention identity");
|
|
467
|
+
if (inventory.publication_id !== publicationId)
|
|
468
|
+
fail("publication_mismatch", "Checkpoint data belongs to another immutable publication");
|
|
469
|
+
if (inventory.base.scope.provider_id !== trustedScope.provider_id || inventory.base.scope.tenant_id !== trustedScope.tenant_id)
|
|
470
|
+
fail("scope_mismatch", "Checkpoint data belongs to another provider or tenant");
|
|
471
|
+
const base = await this.getRoot(inventory.base.root_id);
|
|
472
|
+
if (identity(base.root) !== identity(inventory.base.root))
|
|
473
|
+
fail("base_mismatch", "Checkpoint base differs from its immutable Cloud publication");
|
|
474
|
+
let previous;
|
|
475
|
+
try {
|
|
476
|
+
previous = await this.getRoot(publicationId);
|
|
477
|
+
}
|
|
478
|
+
catch (error) {
|
|
479
|
+
if (!(error instanceof FabricRemoteError) || error.status !== 404)
|
|
480
|
+
throw error;
|
|
481
|
+
}
|
|
482
|
+
if (previous && identity(previous.root) !== identity(inventory.publication))
|
|
483
|
+
fail("publication_mismatch", "Checkpoint publication already refers to different content");
|
|
484
|
+
const transfer = previous ? { receipt: previous, uploaded_objects: 0, uploaded_bytes: 0 } : await this.uploadInventory(publicationId, inventory.publication, inventory.objects, source);
|
|
485
|
+
result = { format_version: 1, kind: "partial_checkpoint_data", source_role: "quiesced_source_complement_data", publication_id: publicationId,
|
|
486
|
+
publication: inventory.publication, runnable: inventory.runnable, q: inventory.q, bundle: transfer.receipt,
|
|
487
|
+
uploaded_objects: transfer.uploaded_objects, uploaded_bytes: transfer.uploaded_bytes,
|
|
488
|
+
...(transfer.cleanup_warning ? { cleanup_warning: transfer.cleanup_warning } : {}) };
|
|
489
|
+
}
|
|
490
|
+
catch (error) {
|
|
491
|
+
failure = error instanceof FabricRemoteError ? error : new FabricRemoteError("source_failed", "Checkpoint data transfer failed");
|
|
492
|
+
}
|
|
493
|
+
finally {
|
|
494
|
+
try {
|
|
495
|
+
await source.unpin(issuedPin);
|
|
496
|
+
}
|
|
497
|
+
catch {
|
|
498
|
+
if (result)
|
|
499
|
+
result.cleanup_warning = [result.cleanup_warning, "Checkpoint data published; temporary local retention needs cleanup"].filter(Boolean).join("; ");
|
|
500
|
+
else
|
|
501
|
+
failure ??= new FabricRemoteError("source_cleanup_failed", "Temporary checkpoint retention needs cleanup");
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
if (failure)
|
|
505
|
+
throw failure;
|
|
506
|
+
return result;
|
|
507
|
+
}
|
|
508
|
+
async uploadRoot(root, object, source) {
|
|
509
|
+
rootId(root);
|
|
510
|
+
validateFabricReference(object);
|
|
511
|
+
let ownedPin;
|
|
512
|
+
try {
|
|
513
|
+
ownedPin = await source.pin(object);
|
|
514
|
+
}
|
|
515
|
+
catch {
|
|
516
|
+
return fail("source_failed", "Could not pin the local transfer source");
|
|
517
|
+
}
|
|
518
|
+
if (typeof ownedPin !== "string" || !ownedPin.trim())
|
|
519
|
+
fail("invalid_source", "Source did not return an owned retention pin");
|
|
520
|
+
let result;
|
|
521
|
+
let failure;
|
|
522
|
+
try {
|
|
523
|
+
const inventory = await source.closure(object);
|
|
524
|
+
if (!Array.isArray(inventory) || inventory.length > 100_000)
|
|
525
|
+
fail("too_large", "Source inventory exceeds object limit");
|
|
526
|
+
const unique = new Map();
|
|
527
|
+
let total = 0;
|
|
528
|
+
for (const value of inventory) {
|
|
529
|
+
const reference = validateFabricReference(value);
|
|
530
|
+
if (!unique.has(identity(reference))) {
|
|
531
|
+
unique.set(identity(reference), reference);
|
|
532
|
+
total += reference.size_bytes;
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
if (!unique.has(identity(object)) || total > 4 * 1024 ** 3)
|
|
536
|
+
fail("invalid_source", "Source inventory is incomplete or exceeds byte limit");
|
|
537
|
+
result = await this.uploadInventory(root, object, [...unique.values()], source);
|
|
270
538
|
}
|
|
271
539
|
catch (error) {
|
|
272
540
|
failure = error instanceof FabricRemoteError ? error : new FabricRemoteError("source_failed", "Pinned source transfer failed");
|
|
@@ -324,6 +592,7 @@ export class FabricRemoteClient {
|
|
|
324
592
|
};
|
|
325
593
|
add(published.root, 0);
|
|
326
594
|
let workerFailure;
|
|
595
|
+
const cancelled = new AbortController();
|
|
327
596
|
let active = 0;
|
|
328
597
|
const waiting = new Set();
|
|
329
598
|
const wake = () => { for (const resolve of waiting)
|
|
@@ -340,15 +609,17 @@ export class FabricRemoteClient {
|
|
|
340
609
|
}
|
|
341
610
|
const { object, depth } = queue[cursor++];
|
|
342
611
|
active++;
|
|
343
|
-
let
|
|
612
|
+
let reservation;
|
|
344
613
|
try {
|
|
345
|
-
|
|
614
|
+
reservation = await this.buffered.acquire(object.size_bytes, this.requestTimeoutMs, cancelled.signal);
|
|
346
615
|
if (workerFailure)
|
|
347
616
|
continue;
|
|
348
617
|
await ensureLease();
|
|
349
618
|
if (workerFailure)
|
|
350
619
|
continue;
|
|
351
|
-
const bytes = await this.
|
|
620
|
+
const bytes = await this.getRetainedObject(object, reservation);
|
|
621
|
+
if (workerFailure)
|
|
622
|
+
continue;
|
|
352
623
|
await ensureLease();
|
|
353
624
|
const children = await target.receive(staging.lease_id, object, bytes);
|
|
354
625
|
if (!Array.isArray(children))
|
|
@@ -361,9 +632,10 @@ export class FabricRemoteClient {
|
|
|
361
632
|
}
|
|
362
633
|
catch (error) {
|
|
363
634
|
workerFailure ??= { error };
|
|
635
|
+
cancelled.abort();
|
|
364
636
|
}
|
|
365
637
|
finally {
|
|
366
|
-
release
|
|
638
|
+
reservation?.release();
|
|
367
639
|
active--;
|
|
368
640
|
wake();
|
|
369
641
|
}
|