@runblackcat/sdk 0.1.0-alpha.56 → 0.1.0-alpha.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -12
- package/package.json +1 -1
- package/sdk/dist/index.d.ts +378 -10
- package/sdk/dist/index.js +62 -1
package/README.md
CHANGED
|
@@ -26,8 +26,8 @@ are in [docs/PRODUCT.md](docs/PRODUCT.md).
|
|
|
26
26
|
rather than a connection.
|
|
27
27
|
- **Queue and backpressure** — a request the fleet has no room for waits durably
|
|
28
28
|
instead of failing, and is placed by a background drain as capacity arrives.
|
|
29
|
-
|
|
30
|
-
|
|
29
|
+
Fleet capacity and the durable queue are the only backpressure; there are no
|
|
30
|
+
per-tenant quotas.
|
|
31
31
|
- **Demand-driven capacity** — the placer sizes the fleet from the backlog it
|
|
32
32
|
cannot place, through a provider seam, and retires idle workers back to zero.
|
|
33
33
|
- **Forkable templates** — build a Docker image into a validated state, memory,
|
|
@@ -49,7 +49,7 @@ are in [docs/PRODUCT.md](docs/PRODUCT.md).
|
|
|
49
49
|
| | |
|
|
50
50
|
|---|---|
|
|
51
51
|
| **Gateway** | Stateless. The single authenticated entry point; resolves sandbox→node and proxies. Scales horizontally. |
|
|
52
|
-
| **Placer** | Single writer. Bin-packs by memory and vCPU, proposes placements, owns
|
|
52
|
+
| **Placer** | Single writer. Bin-packs by memory and vCPU, proposes placements, owns the queue and the node registry. Sole writer of substrate desired capacity. |
|
|
53
53
|
| **Node agent** | One per host. Firecracker lifecycle, jailer, networking, UFFD handler. **Final authority on admission** — the placer proposes, the node accepts or rejects. |
|
|
54
54
|
| **Guest agent** | Inside each sandbox, reached over vsock. Exec, file I/O, template-declared boot services. |
|
|
55
55
|
| **Plugin supervisor (in development)** | Runs operator-installed, digest-pinned job-source plugins outside sandbox hosts. Each installation has its own identity, secret scope and sandbox grant. A small Rust SDK calls the scoped private API. |
|
|
@@ -71,7 +71,7 @@ flowchart TB
|
|
|
71
71
|
subgraph ctrl["Control host"]
|
|
72
72
|
direction LR
|
|
73
73
|
GW["blackcat-gateway<br/>auth · capabilities · routing<br/>listens :443"]
|
|
74
|
-
PL["blackcat-placer<br/>single writer · best-fit<br/>
|
|
74
|
+
PL["blackcat-placer<br/>single writer · best-fit<br/>queue · node registry"]
|
|
75
75
|
DB[("placer.sqlite")]
|
|
76
76
|
GW <-->|"Unix socket"| PL
|
|
77
77
|
PL --- DB
|
|
@@ -158,8 +158,7 @@ construction. Everything else is a network hop.
|
|
|
158
158
|
## Capacity and observability
|
|
159
159
|
|
|
160
160
|
Creation is admitted durably and answered immediately. A request the fleet cannot
|
|
161
|
-
place becomes a **queued acquisition** — it keeps its sandbox identity and
|
|
162
|
-
quota charge, and a background drain places it as capacity appears. Callers poll
|
|
161
|
+
place becomes a **queued acquisition** — it keeps its sandbox identity, and a background drain places it as capacity appears. Callers poll
|
|
163
162
|
the claim they already hold:
|
|
164
163
|
|
|
165
164
|
```
|
|
@@ -169,14 +168,27 @@ GET /v1/sandboxes/acquisitions/{id} -> 202 pending … -> 200 {"state":"p
|
|
|
169
168
|
POST /v1/sandboxes/{id}/renew {ttl_secs} -> 200 {"expires_at":…, "renewable_until":…}
|
|
170
169
|
```
|
|
171
170
|
|
|
171
|
+
To start many identical sandboxes, add `"count": N` (1 to 1000) to the create.
|
|
172
|
+
`request_id` then names the batch and member `i` is `request_id.i`. The whole batch
|
|
173
|
+
is recorded in one durable transaction and answered `202` at once, with no
|
|
174
|
+
connection held through restores; a retry returns the same members.
|
|
175
|
+
|
|
176
|
+
```
|
|
177
|
+
POST /v1/sandboxes {request_id,template,count,lease:{…}}
|
|
178
|
+
-> 202 {"request_id":…, "count":N, "counts":{"pending":N}, "members":[…]}
|
|
179
|
+
GET /v1/sandboxes/batches/{request_id} -> 202 while any member is pending, then 200
|
|
180
|
+
GET /v1/sandboxes/acquisitions/{id}.{i} -> one member, as an ordinary claim
|
|
181
|
+
```
|
|
182
|
+
|
|
172
183
|
`request_id` is caller-supplied and is the idempotency boundary, so repeating a
|
|
173
184
|
create is always safe: it returns the same claim rather than allocating a second
|
|
174
185
|
sandbox.
|
|
175
186
|
|
|
176
|
-
Backpressure
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
187
|
+
Backpressure is fleet capacity and the durable queue; there are no per-tenant
|
|
188
|
+
quotas. A queued acquisition that waits past the queue deadline is torn down by
|
|
189
|
+
the ordinary expiry reconciler. Request-rate limits are operator-set fleet state
|
|
190
|
+
(`blackcat admin rate-limits get|set`): token mint and OIDC limits are on by
|
|
191
|
+
default, every workload limit is unlimited until set.
|
|
180
192
|
|
|
181
193
|
The placer sizes the fleet from that backlog through the `CapacityProvider`
|
|
182
194
|
seam, and is the **only** writer of substrate desired capacity — it raises the
|
|
@@ -194,7 +206,7 @@ blackcat_demand_unsatisfied{tenant="…"} queued acquisitions
|
|
|
194
206
|
blackcat_demand_oldest_age_seconds longest current wait
|
|
195
207
|
blackcat_capacity_target_nodes workers last asked for
|
|
196
208
|
blackcat_nodes{health="healthy"} workers actually registered
|
|
197
|
-
blackcat_placement_rejected_total{reason="…"} no_capacity
|
|
209
|
+
blackcat_placement_rejected_total{reason="…"} no_capacity
|
|
198
210
|
```
|
|
199
211
|
|
|
200
212
|
A fleet pinned at its ceiling shows up as `blackcat_capacity_target_nodes`
|
|
@@ -286,7 +298,8 @@ Linux x86-64/arm64, run `curl -fsSL https://runblackcat.dev/install.sh | sh`;
|
|
|
286
298
|
see [CLI installation](docs/cli-install.md). The installer uses release
|
|
287
299
|
checksums from the public
|
|
288
300
|
[`blackcat-releases`](https://github.com/TreebeardHQ/blackcat-releases) mirror
|
|
289
|
-
and needs neither Docker nor Rust on the client.
|
|
301
|
+
and needs neither Docker nor Rust on the client. To run a fleet on plain
|
|
302
|
+
Debian or Ubuntu servers, see [Bare mode](docs/bare-mode.md). The site itself lives in
|
|
290
303
|
[`site/`](site/) and publishes an allowlisted subset of `docs/`.
|
|
291
304
|
`blackcat template list` shows launchable templates to members without their
|
|
292
305
|
source; `blackcat sandbox claim alpine-e2e-matched` resolves the latest version
|
package/package.json
CHANGED
package/sdk/dist/index.d.ts
CHANGED
|
@@ -21,6 +21,10 @@ export declare const CAPABILITY_OPS: {
|
|
|
21
21
|
readonly RENEW: 4096;
|
|
22
22
|
};
|
|
23
23
|
export type CapabilityOps = number;
|
|
24
|
+
export declare const BLACKCAT_ERROR_CODES: {
|
|
25
|
+
readonly SANDBOX_TRANSITIONING: "sandbox_transitioning";
|
|
26
|
+
readonly FLEET_UPGRADE_QUIESCING: "fleet_upgrade_quiescing";
|
|
27
|
+
};
|
|
24
28
|
export type FetchFunction = (input: RequestInfo | URL, init?: RequestInit & {
|
|
25
29
|
tls?: {
|
|
26
30
|
ca: string;
|
|
@@ -48,6 +52,17 @@ export interface TemplateDefaults {
|
|
|
48
52
|
memory_mib: number;
|
|
49
53
|
hibernate?: HibernateMode;
|
|
50
54
|
on_interruption?: "evacuate" | "destroy";
|
|
55
|
+
/** MiB of virtio-mem region booted into the checkpoint; a multiple of 128. Defaults to the template's memory. */
|
|
56
|
+
hotplug_max_mib?: number;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* The size a stored template version carries: `memory_mib` is the size it was
|
|
60
|
+
* captured at (boot memory plus what was plugged), and a sandbox can later
|
|
61
|
+
* take any size from `base_memory_mib` to `base_memory_mib + hotplug_max_mib`.
|
|
62
|
+
*/
|
|
63
|
+
export interface TemplateShapeDefaults extends TemplateDefaults {
|
|
64
|
+
base_memory_mib: number;
|
|
65
|
+
hotplug_max_mib: number;
|
|
51
66
|
}
|
|
52
67
|
export type HibernateMode = "memory";
|
|
53
68
|
export type TemplateEgressPolicy = {
|
|
@@ -95,6 +110,9 @@ interface TemplateDockerSourceBase {
|
|
|
95
110
|
defaults: TemplateDefaults;
|
|
96
111
|
egress: TemplateEgressPolicy;
|
|
97
112
|
restore_health: TemplateRestoreHealth;
|
|
113
|
+
/** Runs in the guest after it is ready and before the capture, so every
|
|
114
|
+
* clone restores with the app's caches and JIT already hot. Must exit 0. */
|
|
115
|
+
warmup?: TemplateRestoreHealth;
|
|
98
116
|
}
|
|
99
117
|
interface TemplatePreparation {
|
|
100
118
|
parameters: Record<string, {
|
|
@@ -133,8 +151,11 @@ export interface UploadedImage {
|
|
|
133
151
|
unchanged: boolean;
|
|
134
152
|
}
|
|
135
153
|
export interface TemplateSandboxSource {
|
|
154
|
+
/** `warmup` runs in the sandbox just before it is captured; omitted, the
|
|
155
|
+
* sandbox's template's own warm-up (if any) runs. */
|
|
136
156
|
sandbox: {
|
|
137
157
|
id: string;
|
|
158
|
+
warmup?: TemplateRestoreHealth;
|
|
138
159
|
};
|
|
139
160
|
/**
|
|
140
161
|
* A promoted sandbox keeps the machine dimensions it was captured on, so
|
|
@@ -144,20 +165,33 @@ export interface TemplateSandboxSource {
|
|
|
144
165
|
egress?: TemplateEgressPolicy;
|
|
145
166
|
}
|
|
146
167
|
export type TemplateSource = TemplateDockerSource | TemplateUploadedImageSource | TemplateSandboxSource;
|
|
168
|
+
/** What a promotion adds to a template create: how it is warmed and packed. */
|
|
169
|
+
export interface PromotionOptions {
|
|
170
|
+
/** Warm the new version onto this many more nodes in bulk, beyond the node
|
|
171
|
+
* that captured it (at most 99), or onto `"all"` the other healthy nodes.
|
|
172
|
+
* Nodes bound to a capacity reservation that follows the template are warmed
|
|
173
|
+
* regardless. */
|
|
174
|
+
prefetch_copies?: number | "all";
|
|
175
|
+
/** Millicores each clone of the new version reserves, at most 1000 per vCPU.
|
|
176
|
+
* Omitted, the version inherits what the sandbox's own template reserves. */
|
|
177
|
+
cpu_reservation_mcpu?: number;
|
|
178
|
+
}
|
|
147
179
|
export type CreateTemplateInput = {
|
|
148
180
|
name: string;
|
|
149
181
|
source: TemplateDockerSource;
|
|
150
182
|
} | {
|
|
151
183
|
name: string;
|
|
152
184
|
source: TemplateUploadedImageSource;
|
|
153
|
-
} | {
|
|
185
|
+
} | ({
|
|
154
186
|
name?: string;
|
|
155
187
|
source: TemplateSandboxSource;
|
|
156
|
-
};
|
|
188
|
+
} & PromotionOptions);
|
|
157
189
|
export interface Template {
|
|
158
190
|
id: string;
|
|
159
191
|
name: string;
|
|
160
192
|
version: number;
|
|
193
|
+
/** `local_ready` while a promotion's chunks are still being published. */
|
|
194
|
+
state?: "local_ready" | "published";
|
|
161
195
|
/** The gateway stores the complete source manifest as JSON text. */
|
|
162
196
|
source: string;
|
|
163
197
|
current: boolean;
|
|
@@ -187,27 +221,150 @@ export interface PrepareSandboxResponse {
|
|
|
187
221
|
export interface CreateTemplateResponse {
|
|
188
222
|
id: string;
|
|
189
223
|
version: number;
|
|
224
|
+
/**
|
|
225
|
+
* `local_ready`: a promotion whose capture has been proved but whose chunks
|
|
226
|
+
* are still being published. The version is current and claimable, but only
|
|
227
|
+
* on the node that captured it; clones depend only on that node's files and
|
|
228
|
+
* are destroyed if the publication fails. `published`: every chunk is in the
|
|
229
|
+
* object store.
|
|
230
|
+
*/
|
|
231
|
+
state?: "local_ready" | "published";
|
|
232
|
+
/**
|
|
233
|
+
* Capture/publish phase breakdown for a checkpoint promotion, when this
|
|
234
|
+
* create captured a running sandbox. Absent for a template created without
|
|
235
|
+
* a capture, and for a server that predates this field.
|
|
236
|
+
*/
|
|
237
|
+
capture_timing?: {
|
|
238
|
+
capture_ms: number;
|
|
239
|
+
/** How long the source VM stood paused: the memory/state dump, the disk
|
|
240
|
+
* freeze and the resume. Sealing and upload happen after it. */
|
|
241
|
+
pause_ms: number;
|
|
242
|
+
/** Absent while the version is `local_ready`: publication is not over. */
|
|
243
|
+
publish_ms?: number;
|
|
244
|
+
/** Restoring a disposable clone from the capture and running the
|
|
245
|
+
* template's restore-health command in it; both overlap the upload. */
|
|
246
|
+
validation_restore_ms?: number;
|
|
247
|
+
validation_health_ms?: number;
|
|
248
|
+
/** Whether the capturing node was left holding the new version's memory
|
|
249
|
+
* and disk for fault-free restores. */
|
|
250
|
+
warmed?: {
|
|
251
|
+
memory: boolean;
|
|
252
|
+
disk: boolean;
|
|
253
|
+
};
|
|
254
|
+
};
|
|
255
|
+
/** Present for a promotion that asked for or achieved warming. */
|
|
256
|
+
warming?: {
|
|
257
|
+
capturing_node_warm: boolean;
|
|
258
|
+
prefetch_copies: number;
|
|
259
|
+
/** Pass to `templates.prefetchStatus` to follow the extra copies. */
|
|
260
|
+
prefetch_id?: string;
|
|
261
|
+
error?: string;
|
|
262
|
+
};
|
|
190
263
|
}
|
|
191
|
-
export type CreateSandboxInput =
|
|
192
|
-
template: string;
|
|
193
|
-
vcpus?: never;
|
|
194
|
-
memory_mib?: never;
|
|
195
|
-
} | {
|
|
264
|
+
export type CreateSandboxInput = {
|
|
196
265
|
template: string;
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
266
|
+
/** Must equal the checkpoint's vCPUs, which are fixed at capture; omit it. */
|
|
267
|
+
vcpus?: number;
|
|
268
|
+
/**
|
|
269
|
+
* Guest memory in MiB: the checkpoint's captured size or larger, up to its
|
|
270
|
+
* boot memory plus hotplug region (the extra is plugged right after restore).
|
|
271
|
+
*/
|
|
272
|
+
memory_mib?: number;
|
|
273
|
+
/** CPU entitlement in millicores, at most the checkpoint's vCPUs * 1000. */
|
|
274
|
+
cpu_mcpu?: number;
|
|
200
275
|
/** Stable across retries of one acquisition; never reuse for another request. */
|
|
201
276
|
request_id: string;
|
|
202
277
|
ssh_public_key?: string;
|
|
203
278
|
lease: SandboxLeaseInput;
|
|
279
|
+
/** A capacity reservation this create should draw a bound slot from. */
|
|
280
|
+
reservation?: string;
|
|
281
|
+
/**
|
|
282
|
+
* Ask for a capability scoped to each sandbox this request creates, returned
|
|
283
|
+
* in the response (`capability` on the receipt, or on each batch member), so
|
|
284
|
+
* a caller driving many sandboxes mints none of its own. `ops` must be within
|
|
285
|
+
* the creating token's; `ttl_secs` is capped at 900 and renewable like any
|
|
286
|
+
* token. Only the request that creates a sandbox gets its capability: a
|
|
287
|
+
* repeat of the same `request_id` returns none.
|
|
288
|
+
*/
|
|
289
|
+
capability?: {
|
|
290
|
+
ops: string[] | CapabilityOps;
|
|
291
|
+
ttl_secs: number;
|
|
292
|
+
};
|
|
293
|
+
};
|
|
294
|
+
/** A capability scoped to one sandbox, as returned by a create. */
|
|
295
|
+
export interface SandboxCapability {
|
|
296
|
+
token: string;
|
|
297
|
+
token_id: string;
|
|
298
|
+
ops: string[];
|
|
299
|
+
/** Epoch milliseconds. */
|
|
300
|
+
expires_at: number;
|
|
301
|
+
renewable_until: number;
|
|
302
|
+
}
|
|
303
|
+
/** A batch create: `request_id` names the batch, member `i` is `<request_id>.<i>`. */
|
|
304
|
+
export type CreateSandboxBatchInput = CreateSandboxInput & {
|
|
305
|
+
/** How many identical sandboxes to create, 1 to 1000. */
|
|
306
|
+
count: number;
|
|
204
307
|
};
|
|
308
|
+
export type SandboxBatchMemberState = "pending" | "placed" | "failed" | "destroyed";
|
|
309
|
+
export interface SandboxBatchMember {
|
|
310
|
+
/** Poll this member alone with `sandboxes.acquisition(request_id)`. */
|
|
311
|
+
request_id: string;
|
|
312
|
+
id: string;
|
|
313
|
+
state: SandboxBatchMemberState;
|
|
314
|
+
/** Present when the create asked for `capability`. */
|
|
315
|
+
capability?: SandboxCapability;
|
|
316
|
+
}
|
|
317
|
+
export interface SandboxBatch {
|
|
318
|
+
/** The batch's request id. */
|
|
319
|
+
request_id: string;
|
|
320
|
+
count: number;
|
|
321
|
+
/** Members in each state; a state with no members is absent. */
|
|
322
|
+
counts: Partial<Record<SandboxBatchMemberState, number>>;
|
|
323
|
+
members: SandboxBatchMember[];
|
|
324
|
+
}
|
|
205
325
|
export interface SandboxLeaseInput {
|
|
206
326
|
/** Initial lease duration, between 60 seconds and 24 hours. */
|
|
207
327
|
ttl_secs: number;
|
|
208
328
|
/** Hard lifetime from acquisition, at least ttl_secs and at most 30 days. */
|
|
209
329
|
renewable_for_secs: number;
|
|
210
330
|
}
|
|
331
|
+
export type CreateCapacityReservationInput = ({
|
|
332
|
+
/** One immutable template version, pinned for the reservation's life. */
|
|
333
|
+
template: string;
|
|
334
|
+
template_resource?: never;
|
|
335
|
+
} | {
|
|
336
|
+
/** A template resource name: the reservation follows whichever version
|
|
337
|
+
* is current, even one that does not exist yet, and its bound nodes are
|
|
338
|
+
* warmed with each new version as it is activated. */
|
|
339
|
+
template_resource: string;
|
|
340
|
+
template?: never;
|
|
341
|
+
}) & {
|
|
342
|
+
/** How many sandboxes of this template/shape to reserve. */
|
|
343
|
+
count: number;
|
|
344
|
+
vcpus: number;
|
|
345
|
+
memory_mib: number;
|
|
346
|
+
/** How long to hold the reservation, between 1 second and 4 hours. */
|
|
347
|
+
ttl_secs: number;
|
|
348
|
+
};
|
|
349
|
+
export interface CapacityReservation {
|
|
350
|
+
id: string;
|
|
351
|
+
state: "active" | "released" | "expired";
|
|
352
|
+
/** The version nodes are warmed with; null while a reservation following a
|
|
353
|
+
* template resource waits for the resource's first version. */
|
|
354
|
+
template: string | null;
|
|
355
|
+
/** The template resource this reservation follows, or null when pinned. */
|
|
356
|
+
template_resource: string | null;
|
|
357
|
+
count: number;
|
|
358
|
+
vcpus: number;
|
|
359
|
+
memory_mib: number;
|
|
360
|
+
/** Slots consumed by an actual sandbox create. */
|
|
361
|
+
claimed: number;
|
|
362
|
+
/** Slots bound to a specific node and not yet claimed. */
|
|
363
|
+
bound: number;
|
|
364
|
+
/** Bound slots whose node already has the template warm. */
|
|
365
|
+
warm: number;
|
|
366
|
+
expires_at: SystemTimeValue;
|
|
367
|
+
}
|
|
211
368
|
/**
|
|
212
369
|
* Response returned by POST /v1/sandboxes (the gateway has no template field).
|
|
213
370
|
* The gateway sends a `failed` receipt with HTTP 409, so `create` and
|
|
@@ -221,6 +378,14 @@ export type CreateSandboxResponse = {
|
|
|
221
378
|
tier: "boot" | "warm_pool" | "restore";
|
|
222
379
|
expires_at: SystemTimeValue;
|
|
223
380
|
renewable_until: SystemTimeValue;
|
|
381
|
+
/**
|
|
382
|
+
* Phase-timing breakdown for the launch that placed this sandbox, once one
|
|
383
|
+
* has committed. Absent while still queued/pending, and for a server that
|
|
384
|
+
* predates this field.
|
|
385
|
+
*/
|
|
386
|
+
restore_timing?: RestoreTiming;
|
|
387
|
+
/** Present when the create asked for `capability` and created this sandbox. */
|
|
388
|
+
capability?: SandboxCapability;
|
|
224
389
|
} & ({
|
|
225
390
|
state: "placed";
|
|
226
391
|
node: string;
|
|
@@ -232,19 +397,96 @@ export type CreateSandboxResponse = {
|
|
|
232
397
|
node: null;
|
|
233
398
|
error: string;
|
|
234
399
|
});
|
|
400
|
+
/**
|
|
401
|
+
* Restore phase breakdown a node reported for a checkpoint launch (Proposal
|
|
402
|
+
* F, docs/restore-performance.md). `phase_ms` mirrors the node's own
|
|
403
|
+
* `checkpoint vm timing` log line; `chunk_source` reports how much of the
|
|
404
|
+
* checkpoint's memory/disk was fetched remotely vs. served locally, how many
|
|
405
|
+
* UFFD faults and NBD reads this restore actually served (both zero on the
|
|
406
|
+
* file-restore path), and the checkpoint's logical sizes. `memory_source` is
|
|
407
|
+
* `"file"` when a node-local materialized copy let this restore skip
|
|
408
|
+
* UFFD/NBD lazy faulting entirely (fault-free warm restore,
|
|
409
|
+
* docs/restore-performance.md), or `"uffd"` for the lazy chunk-source path
|
|
410
|
+
* every checkpoint restore used before it.
|
|
411
|
+
*/
|
|
412
|
+
export interface RestoreTiming {
|
|
413
|
+
proposal_ms: number;
|
|
414
|
+
restore_vm_ms: number;
|
|
415
|
+
commit_ms: number;
|
|
416
|
+
total_ms: number;
|
|
417
|
+
phase_ms: {
|
|
418
|
+
open: number;
|
|
419
|
+
network: number;
|
|
420
|
+
firecracker: number;
|
|
421
|
+
state: number;
|
|
422
|
+
disk: number;
|
|
423
|
+
pager: number;
|
|
424
|
+
load: number;
|
|
425
|
+
activation: number;
|
|
426
|
+
/** Milliseconds from the TAP being raised until the guest's gateway and resolver answered; absent from a node that predates it, 0 from a guest that does. */
|
|
427
|
+
network_ready?: number;
|
|
428
|
+
};
|
|
429
|
+
memory_source?: "file" | "uffd";
|
|
430
|
+
chunk_source: {
|
|
431
|
+
remote_requests: number;
|
|
432
|
+
remote_bytes: number;
|
|
433
|
+
local_requests?: number;
|
|
434
|
+
local_bytes?: number;
|
|
435
|
+
uffd_faults?: number;
|
|
436
|
+
uffd_fault_bytes?: number;
|
|
437
|
+
nbd_reads?: number;
|
|
438
|
+
nbd_read_bytes?: number;
|
|
439
|
+
logical_memory_bytes: number;
|
|
440
|
+
logical_disk_bytes: number;
|
|
441
|
+
};
|
|
442
|
+
}
|
|
235
443
|
export interface Sandbox {
|
|
236
444
|
id: string;
|
|
237
445
|
template: string;
|
|
238
446
|
state: string;
|
|
239
447
|
node?: string;
|
|
448
|
+
/** vCPUs the guest has now: the template's, until a cold resize reboots it onto another count. */
|
|
449
|
+
vcpus: number;
|
|
450
|
+
/** Current guest memory in MiB (boot memory plus what is plugged). */
|
|
451
|
+
memory_mib: number;
|
|
452
|
+
/** Current CPU entitlement in millicores. */
|
|
453
|
+
cpu_mcpu: number;
|
|
240
454
|
expires_at: SystemTimeValue;
|
|
241
455
|
renewable_until: SystemTimeValue;
|
|
242
456
|
}
|
|
457
|
+
/** A short-lived SSH key to authorize on a running sandbox. */
|
|
458
|
+
export interface GrantSshKeyInput {
|
|
459
|
+
/** One `ssh-ed25519` public key line. */
|
|
460
|
+
public_key: string;
|
|
461
|
+
/** Lifetime in seconds, 1 to 86400. The key only matters while connecting. */
|
|
462
|
+
ttl_secs: number;
|
|
463
|
+
}
|
|
464
|
+
export interface SshKeyGrant {
|
|
465
|
+
/** Unix milliseconds at which the key stops authenticating. */
|
|
466
|
+
expires_at: number;
|
|
467
|
+
/** The login account in the sandbox. */
|
|
468
|
+
user: string;
|
|
469
|
+
/** The sandbox's SSH host public key; pin it before connecting. */
|
|
470
|
+
host_key: string;
|
|
471
|
+
}
|
|
243
472
|
export interface RenewSandboxResponse {
|
|
244
473
|
id: string;
|
|
245
474
|
expires_at: SystemTimeValue;
|
|
246
475
|
renewable_until: SystemTimeValue;
|
|
247
476
|
}
|
|
477
|
+
/** The CPU a node runs on, as it reports it. */
|
|
478
|
+
export interface CpuIdentity {
|
|
479
|
+
vendor: string;
|
|
480
|
+
model_name: string;
|
|
481
|
+
family: number;
|
|
482
|
+
model: number;
|
|
483
|
+
stepping: number;
|
|
484
|
+
microcode: string;
|
|
485
|
+
/** Hash of the CPUID feature leaves a guest's CPUID is derived from. */
|
|
486
|
+
features_hash: string;
|
|
487
|
+
/** The host is itself a guest (nested virtualization). */
|
|
488
|
+
nested: boolean;
|
|
489
|
+
}
|
|
248
490
|
/**
|
|
249
491
|
* Tenant-scoped view returned by GET /v1/fleet. Residents and cached snapshots
|
|
250
492
|
* are already filtered by the gateway to the caller's tenant.
|
|
@@ -262,6 +504,78 @@ export interface FleetNode {
|
|
|
262
504
|
residents: string[];
|
|
263
505
|
cached_snapshots: string[];
|
|
264
506
|
interruption: FleetInterruption | null;
|
|
507
|
+
/** Charged versus measured memory and the node's pressure state. */
|
|
508
|
+
memory: FleetNodeMemory;
|
|
509
|
+
/** Whether the host runs with SMT active. */
|
|
510
|
+
smt_active: boolean;
|
|
511
|
+
/** The node's CPU; absent from nodes older than CPU-aware placement. */
|
|
512
|
+
cpu?: CpuIdentity | null;
|
|
513
|
+
/** 1000 charges file-backed clones in full; 2000 charges them at half. */
|
|
514
|
+
max_memory_ratio_milli: number;
|
|
515
|
+
max_vcpu_ratio_milli: number;
|
|
516
|
+
/** How the fleet decides what a sandbox costs. */
|
|
517
|
+
packing_mode?: "learned" | "manual";
|
|
518
|
+
}
|
|
519
|
+
export type MemoryPressure = "normal" | "high" | "critical";
|
|
520
|
+
export interface FleetNodeMemory {
|
|
521
|
+
/** What admission charged for the node's residents. */
|
|
522
|
+
charged_mib: number;
|
|
523
|
+
/** The residents' sizes before the overcommit discount. */
|
|
524
|
+
plugged_mib: number;
|
|
525
|
+
/** Resident memory of the residents' VMMs, shared pages counted per mapper. */
|
|
526
|
+
measured_rss_mib: number;
|
|
527
|
+
/** Proportional memory the residents cost; compare with `charged_mib`. */
|
|
528
|
+
measured_pss_mib: number;
|
|
529
|
+
/** Memory only one resident maps (Private_Clean + Private_Dirty). */
|
|
530
|
+
measured_uss_mib: number;
|
|
531
|
+
host_available_mib: number;
|
|
532
|
+
psi_some_avg10_milli: number;
|
|
533
|
+
pressure: MemoryPressure;
|
|
534
|
+
}
|
|
535
|
+
export type FleetTrust = "single_trust" | "multi_tenant";
|
|
536
|
+
export interface ResizeSandboxInput {
|
|
537
|
+
/**
|
|
538
|
+
* Target vCPU count. A count different from the sandbox's reboots the guest
|
|
539
|
+
* cold on its own disk: every process ends, template boot services restart,
|
|
540
|
+
* processes started by hand do not.
|
|
541
|
+
*/
|
|
542
|
+
vcpus?: number;
|
|
543
|
+
/** Target guest memory in MiB: boot memory plus a multiple of 128. */
|
|
544
|
+
memory_mib?: number;
|
|
545
|
+
/** Target CPU entitlement in millicores. */
|
|
546
|
+
cpu_mcpu?: number;
|
|
547
|
+
/** Drop the guest's page caches before a shrink. Defaults to true. */
|
|
548
|
+
drop_caches?: boolean;
|
|
549
|
+
/** Stable across retries of one resize; generated when omitted. */
|
|
550
|
+
request_id?: string;
|
|
551
|
+
}
|
|
552
|
+
/** The size a resize reached. A shrink the guest could not fully honour reports what it reached. */
|
|
553
|
+
export interface SandboxResize {
|
|
554
|
+
id: string;
|
|
555
|
+
state: "placed";
|
|
556
|
+
request_id: string;
|
|
557
|
+
/** True when the guest was rebooted to change its vCPU count. */
|
|
558
|
+
cold: boolean;
|
|
559
|
+
vcpus: number;
|
|
560
|
+
memory_mib: number;
|
|
561
|
+
cpu_mcpu: number;
|
|
562
|
+
/** False when the guest stopped short of the requested memory. */
|
|
563
|
+
reached_target: boolean;
|
|
564
|
+
elapsed_ms: number;
|
|
565
|
+
/** Cold resizes only: durable processes the reboot ended. */
|
|
566
|
+
processes_ended?: number;
|
|
567
|
+
/** Cold resizes only: where the reboot spent its time, in milliseconds. */
|
|
568
|
+
phases?: {
|
|
569
|
+
quiesce_ms: number;
|
|
570
|
+
stop_ms: number;
|
|
571
|
+
boot_ms: number;
|
|
572
|
+
plug_ms: number;
|
|
573
|
+
handshake_ms: number;
|
|
574
|
+
health_ms: number;
|
|
575
|
+
};
|
|
576
|
+
/** Cold resizes only: unexpired SSH grants installed again, and ones that failed. */
|
|
577
|
+
ssh_grants_restored?: number;
|
|
578
|
+
ssh_grants_failed?: number;
|
|
265
579
|
}
|
|
266
580
|
export interface FleetInterruption {
|
|
267
581
|
reason: string;
|
|
@@ -269,6 +583,9 @@ export interface FleetInterruption {
|
|
|
269
583
|
}
|
|
270
584
|
export interface FleetResponse {
|
|
271
585
|
nodes: FleetNode[];
|
|
586
|
+
fleet_trust: FleetTrust;
|
|
587
|
+
/** `native`, or a CPU template (`t2`, `t2s`, `t2cl`, `t2a`). */
|
|
588
|
+
cpu_baseline?: string;
|
|
272
589
|
}
|
|
273
590
|
export interface TemplateRefreshPolicy {
|
|
274
591
|
interval_seconds: number;
|
|
@@ -395,6 +712,8 @@ export interface RenewTokenResponse {
|
|
|
395
712
|
}
|
|
396
713
|
export declare class BlackcatApiError extends Error {
|
|
397
714
|
readonly status: number;
|
|
715
|
+
/** Stable gateway error code when one is provided. */
|
|
716
|
+
readonly code?: string;
|
|
398
717
|
readonly retryAfter?: number;
|
|
399
718
|
readonly requestId?: string;
|
|
400
719
|
readonly details: unknown;
|
|
@@ -406,6 +725,7 @@ declare const MAX_PROCESS_STDIN: number;
|
|
|
406
725
|
declare const MAX_EXEC_STDIN: number;
|
|
407
726
|
export declare function epochMillis(value: unknown, field: string): number;
|
|
408
727
|
export declare function parseCreateSandboxResponse(value: unknown, requestId: string): CreateSandboxResponse;
|
|
728
|
+
export declare function parseSandboxBatch(value: unknown, requestId: string): SandboxBatch;
|
|
409
729
|
export declare function assertSecureBlackcatGatewayUrl(baseUrl: string): void;
|
|
410
730
|
declare class GatewayClient {
|
|
411
731
|
readonly baseUrl: string;
|
|
@@ -476,6 +796,16 @@ declare class FleetResource {
|
|
|
476
796
|
/** List the nodes and this tenant's residents/snapshots visible to INSPECT. */
|
|
477
797
|
list(): Promise<FleetResponse>;
|
|
478
798
|
}
|
|
799
|
+
declare class ReservationsResource {
|
|
800
|
+
private readonly client;
|
|
801
|
+
constructor(client: GatewayClient);
|
|
802
|
+
/** Reserve capacity for a burst of sandboxes of one template/shape. */
|
|
803
|
+
create(input: CreateCapacityReservationInput): Promise<CapacityReservation>;
|
|
804
|
+
get(id: string): Promise<CapacityReservation>;
|
|
805
|
+
/** Release early. Unclaimed slots free immediately; bound nodes then fall
|
|
806
|
+
* back to the normal empty-worker scale-in grace period. */
|
|
807
|
+
release(id: string): Promise<CapacityReservation>;
|
|
808
|
+
}
|
|
479
809
|
declare class SandboxesResource {
|
|
480
810
|
private readonly client;
|
|
481
811
|
constructor(client: GatewayClient);
|
|
@@ -487,10 +817,25 @@ declare class SandboxesResource {
|
|
|
487
817
|
* no automatic retry, new ID, or replacement allocation is synthesized here.
|
|
488
818
|
*/
|
|
489
819
|
create(input: CreateSandboxInput): Promise<CreateSandboxResponse>;
|
|
820
|
+
/**
|
|
821
|
+
* Create `count` identical sandboxes under one request ID. Resolves as soon
|
|
822
|
+
* as the whole batch is durably recorded; nothing waits on a restore.
|
|
823
|
+
* Repeat the exact request to recover a lost response: member IDs are
|
|
824
|
+
* stable. Poll `batch` for counts by state, or a member with `acquisition`.
|
|
825
|
+
*/
|
|
826
|
+
createBatch(input: CreateSandboxBatchInput): Promise<SandboxBatch>;
|
|
827
|
+
/** Counts by state and every member of a batch created with `createBatch`. */
|
|
828
|
+
batch(requestId: string): Promise<SandboxBatch>;
|
|
490
829
|
acquisition(requestId: string): Promise<CreateSandboxResponse>;
|
|
491
830
|
get(id: string): Promise<Sandbox>;
|
|
492
831
|
/** Extend the soft expiry without moving the immutable hard deadline. */
|
|
493
832
|
renew(id: string, ttl_secs: number): Promise<RenewSandboxResponse>;
|
|
833
|
+
/**
|
|
834
|
+
* Authorize `public_key` on a running sandbox for `ttl_secs`, whether or not
|
|
835
|
+
* it was created with a key. Needs the sandbox-scoped exec capability. Connect
|
|
836
|
+
* with `CONNECT /v1/sandboxes/{id}/ssh`, or let `blackcat sandbox ssh` do it.
|
|
837
|
+
*/
|
|
838
|
+
grantSshKey(id: string, input: GrantSshKeyInput): Promise<SshKeyGrant>;
|
|
494
839
|
exec(id: string, input: ExecInput): Promise<ExecResult>;
|
|
495
840
|
execStream(id: string, input: ExecInput): AsyncGenerator<ExecStreamChunk>;
|
|
496
841
|
startProcess(id: string, requestId: string, input: ProcessStartInput): Promise<ProcessRecord>;
|
|
@@ -528,10 +873,32 @@ declare class SandboxesResource {
|
|
|
528
873
|
snapshot(id: string): Promise<{
|
|
529
874
|
template: string;
|
|
530
875
|
}>;
|
|
876
|
+
/**
|
|
877
|
+
* Pause a running sandbox's VM in memory: processes, memory and network
|
|
878
|
+
* state are kept, nothing is captured, and `resume` continues it where it
|
|
879
|
+
* stopped. Memory stays resident and charged (`hibernate` frees it); the CPU
|
|
880
|
+
* entitlement is released. Idempotent. Requests that need the guest are
|
|
881
|
+
* refused while it is paused.
|
|
882
|
+
*/
|
|
883
|
+
pause(id: string): Promise<void>;
|
|
531
884
|
hibernate(id: string, input?: {
|
|
532
885
|
mode: HibernateMode;
|
|
533
886
|
}): Promise<void>;
|
|
887
|
+
/**
|
|
888
|
+
* Change a running sandbox's memory (live, within its checkpoint's hotplug
|
|
889
|
+
* region) and CPU entitlement; processes keep running. A `vcpus` different
|
|
890
|
+
* from the sandbox's is a cold reboot instead (`cold: true`). Repeating the
|
|
891
|
+
* same `request_id` replays the result.
|
|
892
|
+
*/
|
|
893
|
+
resize(id: string, input: ResizeSandboxInput): Promise<SandboxResize>;
|
|
894
|
+
/** Continue a paused sandbox, or restore a hibernated one. */
|
|
534
895
|
resume(id: string): Promise<void>;
|
|
896
|
+
/**
|
|
897
|
+
* Destroy a sandbox. Resolves when its VM is dead and the node has released
|
|
898
|
+
* its capacity, not when the node has finished removing its network device,
|
|
899
|
+
* jail and cgroup: that follows in the background, usually within a second.
|
|
900
|
+
* Destroying a sandbox that is already destroyed succeeds.
|
|
901
|
+
*/
|
|
535
902
|
destroy(id: string): Promise<void>;
|
|
536
903
|
}
|
|
537
904
|
export declare class BlackcatClient extends GatewayClient {
|
|
@@ -541,6 +908,7 @@ export declare class BlackcatClient extends GatewayClient {
|
|
|
541
908
|
readonly sandboxes: SandboxesResource;
|
|
542
909
|
readonly tokens: TokensResource;
|
|
543
910
|
readonly fleet: FleetResource;
|
|
911
|
+
readonly reservations: ReservationsResource;
|
|
544
912
|
constructor(options: BlackcatClientOptions);
|
|
545
913
|
/**
|
|
546
914
|
* Create a client that uses a sandbox capability for resource operations.
|
package/sdk/dist/index.js
CHANGED
|
@@ -14,9 +14,14 @@ var CAPABILITY_OPS = {
|
|
|
14
14
|
PROCESS_CANCEL: 2048,
|
|
15
15
|
RENEW: 4096
|
|
16
16
|
};
|
|
17
|
+
var BLACKCAT_ERROR_CODES = {
|
|
18
|
+
SANDBOX_TRANSITIONING: "sandbox_transitioning",
|
|
19
|
+
FLEET_UPGRADE_QUIESCING: "fleet_upgrade_quiescing"
|
|
20
|
+
};
|
|
17
21
|
|
|
18
22
|
class BlackcatApiError extends Error {
|
|
19
23
|
status;
|
|
24
|
+
code;
|
|
20
25
|
retryAfter;
|
|
21
26
|
requestId;
|
|
22
27
|
details;
|
|
@@ -25,6 +30,11 @@ class BlackcatApiError extends Error {
|
|
|
25
30
|
this.name = "BlackcatApiError";
|
|
26
31
|
this.status = status;
|
|
27
32
|
this.details = details;
|
|
33
|
+
if (typeof details === "object" && details !== null && !Array.isArray(details)) {
|
|
34
|
+
const code = details.code;
|
|
35
|
+
if (typeof code === "string")
|
|
36
|
+
this.code = code;
|
|
37
|
+
}
|
|
28
38
|
this.retryAfter = retryAfter;
|
|
29
39
|
this.requestId = typeof requestId === "string" && requestId.length === 32 && /^[a-f0-9]{32}$/i.test(requestId) ? requestId : undefined;
|
|
30
40
|
}
|
|
@@ -83,6 +93,20 @@ function parseCreateSandboxResponse(value, requestId) {
|
|
|
83
93
|
}
|
|
84
94
|
return value;
|
|
85
95
|
}
|
|
96
|
+
var BATCH_STATES = ["pending", "placed", "failed", "destroyed"];
|
|
97
|
+
function parseSandboxBatch(value, requestId) {
|
|
98
|
+
const malformed = () => new BlackcatProtocolError("sandbox batch is malformed or belongs to another request");
|
|
99
|
+
if (typeof value !== "object" || value === null || Array.isArray(value))
|
|
100
|
+
throw malformed();
|
|
101
|
+
const row = value;
|
|
102
|
+
if (row.request_id !== requestId || typeof row.count !== "number" || typeof row.counts !== "object" || row.counts === null || !Array.isArray(row.members) || row.members.length !== row.count || !row.members.every((member) => {
|
|
103
|
+
const m = member;
|
|
104
|
+
return typeof m === "object" && m !== null && typeof m.request_id === "string" && typeof m.id === "string" && BATCH_STATES.includes(String(m.state));
|
|
105
|
+
})) {
|
|
106
|
+
throw malformed();
|
|
107
|
+
}
|
|
108
|
+
return value;
|
|
109
|
+
}
|
|
86
110
|
function parseMintTokenResponse(value) {
|
|
87
111
|
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
88
112
|
throw new BlackcatProtocolError("token mint response is not an object");
|
|
@@ -395,6 +419,22 @@ class FleetResource {
|
|
|
395
419
|
}
|
|
396
420
|
}
|
|
397
421
|
|
|
422
|
+
class ReservationsResource {
|
|
423
|
+
client;
|
|
424
|
+
constructor(client) {
|
|
425
|
+
this.client = client;
|
|
426
|
+
}
|
|
427
|
+
create(input) {
|
|
428
|
+
return this.client.json("POST", "/v1/capacity-reservations", input, "token");
|
|
429
|
+
}
|
|
430
|
+
get(id) {
|
|
431
|
+
return this.client.json("GET", `/v1/capacity-reservations/${pathSegment(id)}`, undefined, "token");
|
|
432
|
+
}
|
|
433
|
+
release(id) {
|
|
434
|
+
return this.client.json("DELETE", `/v1/capacity-reservations/${pathSegment(id)}`, undefined, "token");
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
|
|
398
438
|
class SandboxesResource {
|
|
399
439
|
client;
|
|
400
440
|
constructor(client) {
|
|
@@ -406,6 +446,12 @@ class SandboxesResource {
|
|
|
406
446
|
async create(input) {
|
|
407
447
|
return parseCreateSandboxResponse(await this.client.json("POST", "/v1/sandboxes", input, "token", false), input.request_id);
|
|
408
448
|
}
|
|
449
|
+
async createBatch(input) {
|
|
450
|
+
return parseSandboxBatch(await this.client.json("POST", "/v1/sandboxes", input, "token", false), input.request_id);
|
|
451
|
+
}
|
|
452
|
+
async batch(requestId) {
|
|
453
|
+
return parseSandboxBatch(await this.client.json("GET", `/v1/sandboxes/batches/${pathSegment(requestId)}`, undefined, "token"), requestId);
|
|
454
|
+
}
|
|
409
455
|
async acquisition(requestId) {
|
|
410
456
|
return parseCreateSandboxResponse(await this.client.json("GET", `/v1/sandboxes/acquisitions/${pathSegment(requestId)}`, undefined, "token"), requestId);
|
|
411
457
|
}
|
|
@@ -415,6 +461,9 @@ class SandboxesResource {
|
|
|
415
461
|
renew(id, ttl_secs) {
|
|
416
462
|
return this.client.json("POST", `/v1/sandboxes/${pathSegment(id)}/renew`, { ttl_secs }, "token");
|
|
417
463
|
}
|
|
464
|
+
grantSshKey(id, input) {
|
|
465
|
+
return this.client.json("POST", `/v1/sandboxes/${pathSegment(id)}/ssh-keys`, input, "token");
|
|
466
|
+
}
|
|
418
467
|
async exec(id, input) {
|
|
419
468
|
return this.client.json("POST", `/v1/sandboxes/${pathSegment(id)}/exec`, input, "token", false, execDeadline(input)).then(execResult);
|
|
420
469
|
}
|
|
@@ -596,11 +645,19 @@ class SandboxesResource {
|
|
|
596
645
|
snapshot(id) {
|
|
597
646
|
return this.client.json("POST", `/v1/sandboxes/${pathSegment(id)}/snapshot`, undefined, "token");
|
|
598
647
|
}
|
|
648
|
+
pause(id) {
|
|
649
|
+
return this.client.raw("POST", `/v1/sandboxes/${pathSegment(id)}/pause`, undefined, "token").then(() => {
|
|
650
|
+
return;
|
|
651
|
+
});
|
|
652
|
+
}
|
|
599
653
|
hibernate(id, input) {
|
|
600
654
|
return this.client.raw("POST", `/v1/sandboxes/${pathSegment(id)}/hibernate`, input === undefined ? undefined : JSON.stringify(input), "token", input === undefined ? undefined : "application/json", false).then(() => {
|
|
601
655
|
return;
|
|
602
656
|
});
|
|
603
657
|
}
|
|
658
|
+
resize(id, input) {
|
|
659
|
+
return this.client.json("POST", `/v1/sandboxes/${pathSegment(id)}/resize`, { ...input, request_id: input.request_id ?? `resize-${crypto.randomUUID()}` }, "token", false);
|
|
660
|
+
}
|
|
604
661
|
resume(id) {
|
|
605
662
|
return this.client.raw("POST", `/v1/sandboxes/${pathSegment(id)}/resume`, undefined, "token").then(() => {
|
|
606
663
|
return;
|
|
@@ -618,6 +675,7 @@ class BlackcatClient extends GatewayClient {
|
|
|
618
675
|
sandboxes;
|
|
619
676
|
tokens;
|
|
620
677
|
fleet;
|
|
678
|
+
reservations;
|
|
621
679
|
constructor(options) {
|
|
622
680
|
if (options.apiKey !== undefined)
|
|
623
681
|
assertSecureBlackcatGatewayUrl(options.baseUrl);
|
|
@@ -628,6 +686,7 @@ class BlackcatClient extends GatewayClient {
|
|
|
628
686
|
this.sandboxes = new SandboxesResource(this);
|
|
629
687
|
this.tokens = new TokensResource(this);
|
|
630
688
|
this.fleet = new FleetResource(this);
|
|
689
|
+
this.reservations = new ReservationsResource(this);
|
|
631
690
|
}
|
|
632
691
|
withToken(token, renewCapability) {
|
|
633
692
|
const capability = typeof token === "string" ? undefined : token;
|
|
@@ -644,6 +703,7 @@ class BlackcatClient extends GatewayClient {
|
|
|
644
703
|
}
|
|
645
704
|
}
|
|
646
705
|
export {
|
|
706
|
+
parseSandboxBatch,
|
|
647
707
|
parseCreateSandboxResponse,
|
|
648
708
|
epochMillis,
|
|
649
709
|
assertSecureBlackcatGatewayUrl,
|
|
@@ -654,5 +714,6 @@ export {
|
|
|
654
714
|
MAX_EXEC_STDIN,
|
|
655
715
|
CAPABILITY_OPS,
|
|
656
716
|
BlackcatClient,
|
|
657
|
-
BlackcatApiError
|
|
717
|
+
BlackcatApiError,
|
|
718
|
+
BLACKCAT_ERROR_CODES
|
|
658
719
|
};
|