@molecule/api-code-sandbox-flyio 1.3.5 → 1.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +295 -32
- package/dist/api.d.ts +17 -2
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +24 -3
- package/dist/ids.d.ts +9 -10
- package/dist/ids.d.ts.map +1 -1
- package/dist/ids.js +32 -18
- package/dist/index.d.ts +33 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +33 -4
- package/dist/lifecycle.d.ts +142 -0
- package/dist/lifecycle.d.ts.map +1 -0
- package/dist/lifecycle.js +14 -0
- package/dist/provider.d.ts +2 -1
- package/dist/provider.d.ts.map +1 -1
- package/dist/provider.js +1494 -306
- package/dist/types.d.ts +33 -7
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +0 -10
- package/package.json +2 -2
package/dist/provider.js
CHANGED
|
@@ -6,16 +6,16 @@
|
|
|
6
6
|
*
|
|
7
7
|
* @module
|
|
8
8
|
*/
|
|
9
|
-
import { createHash } from 'node:crypto';
|
|
9
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
10
10
|
import { promisify } from 'node:util';
|
|
11
11
|
import { gzip } from 'node:zlib';
|
|
12
12
|
import { getLogger } from '@molecule/api-bond';
|
|
13
13
|
import { t } from '@molecule/api-i18n';
|
|
14
|
-
import { FlyApiClient, FlyApiError, normalizeApiUrl } from './api.js';
|
|
14
|
+
import { FlyApiClient, FlyApiError, normalizeApiUrl, retryDelayMs, } from './api.js';
|
|
15
15
|
import { buildEgressProbeCommand, DEFAULT_EGRESS_POLICY_NAME, DEFAULT_EGRESS_PROBE_TARGETS, describePublicReach, extractPolicyId, extractPolicyPorts, parseEgressAllowedPorts, parseEgressProbeTargets, verdictForProbeExit, } from './egress.js';
|
|
16
16
|
import { execCommand, toExecResult } from './exec.js';
|
|
17
17
|
import { assertPrivateRoutesForEnv, decodePrivateRoutes, DEFAULT_PRIVATE_IP_TYPE, encodePrivateRoutes, flycastHost, mergeEgressPorts, parsePrivateServices, PRIVATE_ROUTES_METADATA_SUFFIX, } from './flycast.js';
|
|
18
|
-
import { appNameForProject, mapMachineState, parseSandboxId, toSandboxId } from './ids.js';
|
|
18
|
+
import { appNameForProject, appNameNamespace, mapMachineState, parseSandboxId, toSandboxId, } from './ids.js';
|
|
19
19
|
import { createTemplateStore, MAX_PRESIGN_EXPIRY_SECONDS } from './storage.js';
|
|
20
20
|
import * as templates from './templates.js';
|
|
21
21
|
import { shellQuote } from './utilities.js';
|
|
@@ -102,6 +102,8 @@ const DEFAULT_PREVIEW_PORT = 5173;
|
|
|
102
102
|
const WORKSPACE_PATH = '/workspace';
|
|
103
103
|
/** Default volume size in GB when `SandboxConfig.resources.diskMB` is unset. */
|
|
104
104
|
const DEFAULT_VOLUME_GB = 10;
|
|
105
|
+
/** Exact-identity reads allowed after an ambiguous create before its outcome is declared unknown. */
|
|
106
|
+
const CREATE_RECONCILE_ATTEMPTS = 3;
|
|
105
107
|
/** Page size for the org-wide Machine listing. */
|
|
106
108
|
const LIST_PAGE_SIZE = 200;
|
|
107
109
|
/** Hard cap on pages fetched while listing, so a bad cursor cannot loop forever. */
|
|
@@ -204,13 +206,13 @@ const MACHINE_CREATE_ATTEMPTS = 6;
|
|
|
204
206
|
* means no Machine was created — unlike a 5xx/timeout, where one might have
|
|
205
207
|
* been. 412 is safe for the same reason: "volume already claimed by machine
|
|
206
208
|
* <id>" refuses the create. It is what the create answers in the seconds after
|
|
207
|
-
*
|
|
208
|
-
*
|
|
209
|
-
*
|
|
209
|
+
* an explicit destroy released the Machine that held the volume — Fly releases
|
|
210
|
+
* the claim asynchronously, and an immediate create lost a production deploy
|
|
211
|
+
* to it (2026-10-14; the identical retry a
|
|
210
212
|
* minute later succeeded). Only these two are added on top of the client's
|
|
211
213
|
* default retryable set; neither retry can double-create.
|
|
212
214
|
*/
|
|
213
|
-
const MACHINE_CREATE_RETRY_STATUSES = [404, 412];
|
|
215
|
+
const MACHINE_CREATE_RETRY_STATUSES = [400, 404, 412];
|
|
214
216
|
/**
|
|
215
217
|
* What Fly answers when it reached the exec endpoint but the connection to the
|
|
216
218
|
* Machine's exec agent died before the command ran: `412 {"error":
|
|
@@ -300,6 +302,16 @@ const PROBE_EXEC_HEADROOM_SECONDS = 10;
|
|
|
300
302
|
*/
|
|
301
303
|
class FlyioSandboxProvider {
|
|
302
304
|
name = 'flyio';
|
|
305
|
+
/**
|
|
306
|
+
* Machines this provider runs live until destroyed and each app is reachable
|
|
307
|
+
* at its own routable URL (`https://<app>.fly.dev`, plus the shared private
|
|
308
|
+
* network for the platform's front) — the two things a long-lived hosting
|
|
309
|
+
* role needs, which is why this provider is selectable for it.
|
|
310
|
+
*/
|
|
311
|
+
hostingCapabilities = {
|
|
312
|
+
longLived: true,
|
|
313
|
+
routable: true,
|
|
314
|
+
};
|
|
303
315
|
/**
|
|
304
316
|
* The idle policy a sandbox gets when its own create does not name one — this
|
|
305
317
|
* provider's configured `autostop` (`FLY_PRODUCTION_AUTOSTOP` for production
|
|
@@ -359,8 +371,11 @@ class FlyioSandboxProvider {
|
|
|
359
371
|
* Resolves the Fly region for Machines and volumes.
|
|
360
372
|
* @returns The region code.
|
|
361
373
|
*/
|
|
362
|
-
region() {
|
|
363
|
-
|
|
374
|
+
region(createRegion) {
|
|
375
|
+
// FLY_REGION is the API Machine's own location on Fly, not a tenant
|
|
376
|
+
// placement decision. Only provider configuration or this create's recorded
|
|
377
|
+
// region may choose where new project resources live.
|
|
378
|
+
return createRegion ?? this.config.region ?? DEFAULT_REGION;
|
|
364
379
|
}
|
|
365
380
|
/**
|
|
366
381
|
* Whether each project gets its own Fly app on its own custom 6PN network.
|
|
@@ -470,10 +485,51 @@ class FlyioSandboxProvider {
|
|
|
470
485
|
* `<app>.internal` already resolves and Flycast would add nothing.
|
|
471
486
|
* @returns The declared services, or `undefined` when none are configured.
|
|
472
487
|
*/
|
|
473
|
-
privateServices() {
|
|
488
|
+
privateServices(createRegion) {
|
|
474
489
|
if (!this.appPerProject())
|
|
475
490
|
return undefined;
|
|
476
|
-
|
|
491
|
+
const fallback = this.config.privateServices ?? parsePrivateServices(process.env.FLY_SANDBOX_PRIVATE_SERVICES);
|
|
492
|
+
if (!createRegion || createRegion === DEFAULT_REGION)
|
|
493
|
+
return fallback;
|
|
494
|
+
const suffix = createRegion.replace(/-/g, '_').toUpperCase();
|
|
495
|
+
const variable = `FLY_SANDBOX_PRIVATE_SERVICES_${suffix}`;
|
|
496
|
+
const regional = this.config.privateServicesByRegion?.[createRegion] ??
|
|
497
|
+
parsePrivateServices(process.env[variable]);
|
|
498
|
+
if (!regional && fallback?.length) {
|
|
499
|
+
throw new Error(`No Fly private services are configured for create region "${createRegion}"; ` +
|
|
500
|
+
`set ${variable} (or config.privateServicesByRegion.${createRegion}). ` +
|
|
501
|
+
"Refusing to reuse another region's tenant route.");
|
|
502
|
+
}
|
|
503
|
+
return regional ?? fallback;
|
|
504
|
+
}
|
|
505
|
+
/** Fence and cancel every provisioning mutation at its owning attempt. */
|
|
506
|
+
async requestForAttempt(attempt, path, options) {
|
|
507
|
+
if (attempt.signal.aborted)
|
|
508
|
+
throw attempt.signal.reason;
|
|
509
|
+
await this.config.createAttemptJournal?.assertGeneration(attempt.attemptId, attempt.generation);
|
|
510
|
+
if (attempt.signal.aborted)
|
|
511
|
+
throw attempt.signal.reason;
|
|
512
|
+
const result = await this.client.request(path, { ...options, signal: attempt.signal });
|
|
513
|
+
// A response can arrive after ownership was lost. The write may remain
|
|
514
|
+
// unresolved, but a superseded worker must not journal/adopt its outcome.
|
|
515
|
+
if (attempt.signal.aborted)
|
|
516
|
+
throw attempt.signal.reason;
|
|
517
|
+
await this.config.createAttemptJournal?.assertGeneration(attempt.attemptId, attempt.generation);
|
|
518
|
+
return result;
|
|
519
|
+
}
|
|
520
|
+
/** Authorize a DELETE from the committed retired row on its live lock session. */
|
|
521
|
+
async deleteRetired(intent, generation, path, signal) {
|
|
522
|
+
const journal = this.config.createAttemptJournal;
|
|
523
|
+
if (!journal)
|
|
524
|
+
throw new Error('Retired deletion requires its durable create-attempt journal');
|
|
525
|
+
await journal.authorizeDestruction(intent.attemptId, generation);
|
|
526
|
+
return this.requestForDestruction(path, { method: 'DELETE', nullOn: [404] }, signal);
|
|
527
|
+
}
|
|
528
|
+
/** The sole request dispatch for retirement and settlement, always lock-fenced. */
|
|
529
|
+
async requestForDestruction(path, options, signal) {
|
|
530
|
+
if (signal.aborted)
|
|
531
|
+
throw signal.reason;
|
|
532
|
+
return this.client.request(path, { ...options, signal });
|
|
477
533
|
}
|
|
478
534
|
/**
|
|
479
535
|
* Idempotently ensures the Fly app exists, creating it on its own custom 6PN
|
|
@@ -495,27 +551,52 @@ class FlyioSandboxProvider {
|
|
|
495
551
|
* @returns `routes`: target app name → allocated private address, for
|
|
496
552
|
* recording in the Machine's metadata (empty when no private services are
|
|
497
553
|
* declared); `allocated`: the subset of `routes` this call created, which
|
|
498
|
-
* the caller releases when the boot fails
|
|
499
|
-
*
|
|
500
|
-
* held — only this call's own allocations are safe to take back).
|
|
554
|
+
* the caller releases when the boot fails. The app itself always survives:
|
|
555
|
+
* another create can adopt it by name while this call is still failing.
|
|
501
556
|
*/
|
|
502
557
|
async ensureApp(app, options = {}) {
|
|
503
558
|
const existing = await this.client.request(`/apps/${app}`, { nullOn: [404] });
|
|
504
559
|
let createdHere = false;
|
|
560
|
+
if (existing && options.requireCreated) {
|
|
561
|
+
return { routes: {}, allocated: {}, created: false };
|
|
562
|
+
}
|
|
505
563
|
if (!existing) {
|
|
506
564
|
const body = { name: app, org_slug: this.orgSlug() };
|
|
507
565
|
const network = this.networkFor(app);
|
|
508
566
|
if (network)
|
|
509
567
|
body.network = network;
|
|
510
568
|
let created = true;
|
|
569
|
+
const attempt = options.attempt;
|
|
570
|
+
const journal = this.config.createAttemptJournal;
|
|
571
|
+
if (attempt && journal) {
|
|
572
|
+
// Persist who is attempting the create BEFORE Fly can accept it. The
|
|
573
|
+
// separate accepted bit below prevents a pending/unknown POST from
|
|
574
|
+
// becoming deletion authority after a crash or reconciliation error.
|
|
575
|
+
await journal.recordAppProvenance(attempt.attemptId, attempt.generation, 'pending');
|
|
576
|
+
}
|
|
511
577
|
try {
|
|
512
|
-
|
|
578
|
+
if (attempt) {
|
|
579
|
+
await this.requestForAttempt(attempt, `/apps`, { method: 'POST', body });
|
|
580
|
+
}
|
|
581
|
+
else {
|
|
582
|
+
await this.client.request(`/apps`, { method: 'POST', body });
|
|
583
|
+
}
|
|
584
|
+
// Accepted POST provenance is established before later policy/route
|
|
585
|
+
// steps can fail, so diagnostic cleanup can still retire its own app.
|
|
586
|
+
if (options.ownership)
|
|
587
|
+
options.ownership.created = true;
|
|
588
|
+
if (attempt && journal) {
|
|
589
|
+
await journal.recordAppProvenance(attempt.attemptId, attempt.generation, 'created');
|
|
590
|
+
}
|
|
513
591
|
logger.info('Created Fly sandbox app', { app, network: body.network });
|
|
514
592
|
}
|
|
515
593
|
catch (error) {
|
|
516
594
|
// 409/422 means another concurrent boot created the same app first, which
|
|
517
595
|
// is exactly the outcome we wanted. Anything else is a real failure.
|
|
518
596
|
if (error instanceof FlyApiError && (error.status === 409 || error.status === 422)) {
|
|
597
|
+
if (attempt && journal) {
|
|
598
|
+
await journal.recordAppProvenance(attempt.attemptId, attempt.generation, 'adopted');
|
|
599
|
+
}
|
|
519
600
|
logger.debug('Fly sandbox app already exists', { app });
|
|
520
601
|
created = false;
|
|
521
602
|
}
|
|
@@ -525,45 +606,18 @@ class FlyioSandboxProvider {
|
|
|
525
606
|
}
|
|
526
607
|
createdHere = created;
|
|
527
608
|
}
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
await this.assignSharedIpv4(app);
|
|
531
|
-
await this.applyEgressPolicy(app);
|
|
532
|
-
if (options.privateRoutes === false)
|
|
533
|
-
return { routes: {}, allocated: {}, created: createdHere };
|
|
534
|
-
return {
|
|
535
|
-
...(await this.ensurePrivateRoutes(app, Boolean(existing))),
|
|
536
|
-
created: createdHere,
|
|
537
|
-
};
|
|
609
|
+
if (options.requireCreated && !createdHere) {
|
|
610
|
+
return { routes: {}, allocated: {}, created: false };
|
|
538
611
|
}
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
await this.discardCreatedApp(app, {});
|
|
542
|
-
throw error;
|
|
612
|
+
if (createdHere && this.config.assignSharedIpv4) {
|
|
613
|
+
await this.assignSharedIpv4(app, options.attempt);
|
|
543
614
|
}
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
* failed. Without it every failed create leaves an app behind with its
|
|
548
|
-
* addresses and volume: on 2026-09-28, 34 empty `mol-prod-*` apps had piled
|
|
549
|
-
* up in production, one per deploy that hit the organization's Machine limit
|
|
550
|
-
* (`POST /machines` → 422) after its app was already made. An app that
|
|
551
|
-
* existed before the call is never touched — it may hold a working Machine.
|
|
552
|
-
* @param app - The app this call created.
|
|
553
|
-
* @param privateRoutes - Flycast addresses already allocated for it.
|
|
554
|
-
*/
|
|
555
|
-
async discardCreatedApp(app, privateRoutes) {
|
|
556
|
-
try {
|
|
557
|
-
await this.client.request(`/apps/${app}`, { method: 'DELETE', nullOn: [404] });
|
|
558
|
-
logger.info('Deleted a Fly sandbox app whose create failed', { app });
|
|
615
|
+
await this.applyEgressPolicy(app, options.attempt, options.createRegion);
|
|
616
|
+
if (options.privateRoutes === false) {
|
|
617
|
+
return { routes: {}, allocated: {}, created: createdHere };
|
|
559
618
|
}
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
app,
|
|
563
|
-
error,
|
|
564
|
-
});
|
|
565
|
-
}
|
|
566
|
-
await this.releasePrivateRoutes(privateRoutes);
|
|
619
|
+
const routes = await this.ensurePrivateRoutes(app, Boolean(existing), options.attempt, options.createRegion);
|
|
620
|
+
return { ...routes, created: createdHere };
|
|
567
621
|
}
|
|
568
622
|
/**
|
|
569
623
|
* Allocates a Flycast private address on every declared service app, INTO this
|
|
@@ -594,37 +648,42 @@ class FlyioSandboxProvider {
|
|
|
594
648
|
* the only addresses safe to release when the boot fails after allocation,
|
|
595
649
|
* since the recorded ones belong to Machines an earlier boot may still run.
|
|
596
650
|
*/
|
|
597
|
-
async ensurePrivateRoutes(app, appExisted) {
|
|
598
|
-
const services = this.privateServices();
|
|
651
|
+
async ensurePrivateRoutes(app, appExisted, attempt, createRegion) {
|
|
652
|
+
const services = this.privateServices(createRegion);
|
|
599
653
|
const network = this.networkFor(app);
|
|
600
654
|
if (!services?.length || !network)
|
|
601
655
|
return { routes: {}, allocated: {} };
|
|
602
656
|
const routes = appExisted ? await this.recordedPrivateRoutes(app) : {};
|
|
603
657
|
// The addresses THIS call allocated, tracked apart from `routes` so a
|
|
604
|
-
// failure part-way down the next list can release exactly them.
|
|
605
|
-
//
|
|
606
|
-
//
|
|
607
|
-
// allocated for an earlier service stays behind on its target app, inside a
|
|
608
|
-
// 6PN whose app is gone, and Fly's listing does not report which network an
|
|
609
|
-
// address belongs to: nothing could ever find it there to release it.
|
|
658
|
+
// failure part-way down the next list can release exactly them. These
|
|
659
|
+
// addresses live on OTHER apps, so deleting this project's app would not
|
|
660
|
+
// release them; Fly's listing also cannot reconstruct their target network.
|
|
610
661
|
const allocated = {};
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
662
|
+
for (const service of services) {
|
|
663
|
+
if (routes[service.app])
|
|
664
|
+
continue;
|
|
665
|
+
if (attempt && this.config.createAttemptJournal) {
|
|
666
|
+
await this.config.createAttemptJournal.recordAddress(attempt.attemptId, {
|
|
667
|
+
targetApp: service.app,
|
|
668
|
+
network,
|
|
669
|
+
owned: false,
|
|
670
|
+
resolved: false,
|
|
671
|
+
});
|
|
672
|
+
}
|
|
673
|
+
const ip = await this.allocatePrivateAddress(service, network, attempt);
|
|
674
|
+
if (attempt && this.config.createAttemptJournal) {
|
|
675
|
+
await this.config.createAttemptJournal.recordAddress(attempt.attemptId, {
|
|
676
|
+
targetApp: service.app,
|
|
677
|
+
network,
|
|
678
|
+
...(ip ? { ip } : {}),
|
|
679
|
+
owned: Boolean(ip),
|
|
680
|
+
resolved: true,
|
|
681
|
+
});
|
|
682
|
+
}
|
|
683
|
+
if (ip) {
|
|
684
|
+
routes[service.app] = ip;
|
|
685
|
+
allocated[service.app] = ip;
|
|
620
686
|
}
|
|
621
|
-
}
|
|
622
|
-
catch (error) {
|
|
623
|
-
// Only what this call allocated. An address recorded by an EARLIER boot
|
|
624
|
-
// (`appExisted`) is left alone: the app survived this failure and may
|
|
625
|
-
// still hold a Machine reaching through that route.
|
|
626
|
-
await this.releasePrivateRoutes(allocated);
|
|
627
|
-
throw error;
|
|
628
687
|
}
|
|
629
688
|
return { routes, allocated };
|
|
630
689
|
}
|
|
@@ -657,50 +716,26 @@ class FlyioSandboxProvider {
|
|
|
657
716
|
return {};
|
|
658
717
|
}
|
|
659
718
|
/**
|
|
660
|
-
*
|
|
661
|
-
*
|
|
662
|
-
*
|
|
663
|
-
* retried queue job — leaves a Machine holding the app's single-attach volume.
|
|
664
|
-
* The next create's `POST /machines` then fails with `412 volume already
|
|
665
|
-
* claimed by machine <id>`, and the project can never recover on its own. The
|
|
666
|
-
* app is one-per-project and holds exactly one sandbox, so any managed Machine
|
|
667
|
-
* still here is a stale partial: destroy it (force covers running/suspended) to
|
|
668
|
-
* free the volume and make create idempotent per project.
|
|
669
|
-
* @param app - The Fly app name.
|
|
719
|
+
* Read and normalize Fly's assignment list once for allocation and settlement.
|
|
720
|
+
* Current Fly returns `{ips: [...]}` and a network object; the older array /
|
|
721
|
+
* `ip_assignments` shapes remain readable for compatibility with saved clients.
|
|
670
722
|
*/
|
|
671
|
-
async
|
|
672
|
-
const
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
return;
|
|
685
|
-
}
|
|
686
|
-
for (const machine of Array.isArray(machines) ? machines : []) {
|
|
687
|
-
if (machine.config?.metadata?.[managedKey] !== 'true')
|
|
688
|
-
continue;
|
|
689
|
-
try {
|
|
690
|
-
await this.client.request(`/apps/${app}/machines/${machine.id}?force=true`, {
|
|
691
|
-
method: 'DELETE',
|
|
692
|
-
nullOn: [404],
|
|
693
|
-
});
|
|
694
|
-
logger.info('Destroyed a stale Machine before re-creating the sandbox', {
|
|
695
|
-
app,
|
|
696
|
-
machineId: machine.id,
|
|
697
|
-
});
|
|
698
|
-
}
|
|
699
|
-
catch (error) {
|
|
700
|
-
logger.warn('Failed to destroy a stale Machine before create; create may fail with ' +
|
|
701
|
-
'412 volume already claimed', { app, machineId: machine.id, error });
|
|
723
|
+
async listPrivateAssignments(app, signal) {
|
|
724
|
+
const listed = signal
|
|
725
|
+
? await this.requestForDestruction(`/apps/${app}/ip_assignments`, {}, signal)
|
|
726
|
+
: await this.client.request(`/apps/${app}/ip_assignments`);
|
|
727
|
+
const assignments = Array.isArray(listed) ? listed : (listed?.ips ?? listed?.ip_assignments);
|
|
728
|
+
if (!assignments)
|
|
729
|
+
return null;
|
|
730
|
+
return assignments.map((assignment) => {
|
|
731
|
+
if (typeof assignment.network === 'string')
|
|
732
|
+
return assignment;
|
|
733
|
+
const network = assignment.network;
|
|
734
|
+
if (!network || network.org_slug !== this.orgSlug()) {
|
|
735
|
+
return { ...assignment, network: undefined };
|
|
702
736
|
}
|
|
703
|
-
|
|
737
|
+
return { ...assignment, network: network.name ?? '' };
|
|
738
|
+
});
|
|
704
739
|
}
|
|
705
740
|
/**
|
|
706
741
|
* Allocates ONE Flycast private address on a target app, for this project's
|
|
@@ -718,7 +753,7 @@ class FlyioSandboxProvider {
|
|
|
718
753
|
* address — an allocation this provider can neither verify nor release later
|
|
719
754
|
* is untracked state, and the boot is the last moment it can be reported.
|
|
720
755
|
*/
|
|
721
|
-
async allocatePrivateAddress(service, network) {
|
|
756
|
+
async allocatePrivateAddress(service, network, attempt) {
|
|
722
757
|
let assignment;
|
|
723
758
|
try {
|
|
724
759
|
// NO org_slug here. The ip_assignments endpoint derives the org from the
|
|
@@ -727,25 +762,28 @@ class FlyioSandboxProvider {
|
|
|
727
762
|
// slug that app creation REQUIRES is refused here; omitting it returns 200
|
|
728
763
|
// and allocates the address). app creation still sends org_slug — the two
|
|
729
764
|
// endpoints genuinely differ.
|
|
730
|
-
|
|
765
|
+
const request = {
|
|
731
766
|
method: 'POST',
|
|
732
767
|
body: {
|
|
733
768
|
type: this.config.privateIpAssignmentType ?? DEFAULT_PRIVATE_IP_TYPE,
|
|
734
769
|
network,
|
|
735
770
|
},
|
|
736
|
-
}
|
|
771
|
+
};
|
|
772
|
+
assignment = attempt
|
|
773
|
+
? await this.requestForAttempt(attempt, `/apps/${service.app}/ip_assignments`, request)
|
|
774
|
+
: await this.client.request(`/apps/${service.app}/ip_assignments`, request);
|
|
737
775
|
}
|
|
738
776
|
catch (error) {
|
|
739
|
-
// A conflict
|
|
740
|
-
//
|
|
741
|
-
//
|
|
742
|
-
// failure; it is an address this provider can no longer release, because
|
|
743
|
-
// Fly's listing does not report which network an address belongs to.
|
|
777
|
+
// A conflict can be the duplicate response to an accepted write whose
|
|
778
|
+
// first response was lost. Resolve it by target app AND network, then the
|
|
779
|
+
// caller journals the concrete IP for adoption or later release.
|
|
744
780
|
if (error instanceof FlyApiError && (error.status === 409 || error.status === 422)) {
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
781
|
+
const assignments = await this.listPrivateAssignments(service.app, attempt?.signal);
|
|
782
|
+
const existing = assignments?.find((candidate) => candidate.network === network);
|
|
783
|
+
if (existing?.ip)
|
|
784
|
+
return existing.ip;
|
|
785
|
+
throw new Error(`Fly rejected a private-address allocation for "${service.app}" on "${network}", ` +
|
|
786
|
+
'but no matching assignment could be resolved', { cause: error });
|
|
749
787
|
}
|
|
750
788
|
throw new Error(t('codeSandbox.flyio.error.privateRouteFailed', { app: service.app, network }, {
|
|
751
789
|
defaultValue: `Could not allocate a Flycast private address on Fly app "${service.app}" for ` +
|
|
@@ -771,28 +809,42 @@ class FlyioSandboxProvider {
|
|
|
771
809
|
});
|
|
772
810
|
return assignment.ip;
|
|
773
811
|
}
|
|
812
|
+
/** Validate the exact IP's current network; metadata alone is not DELETE authority. */
|
|
813
|
+
async privateRouteMatchesNetwork(target, ip, network, signal) {
|
|
814
|
+
if (!network)
|
|
815
|
+
return false;
|
|
816
|
+
const assignments = await this.listPrivateAssignments(target, signal);
|
|
817
|
+
return Boolean(assignments?.some((candidate) => candidate.ip === ip && candidate.network === network));
|
|
818
|
+
}
|
|
774
819
|
/**
|
|
775
|
-
* Releases
|
|
776
|
-
*
|
|
777
|
-
* Best-effort: the sandbox is being destroyed either way, and an address left
|
|
778
|
-
* behind is a reconciliation item (`fly ips list --app <target>`) rather than
|
|
779
|
-
* a failed teardown. It routes into a 6PN that no longer has any app in it.
|
|
780
|
-
* @param routes - Target app name → private address, from Machine metadata.
|
|
820
|
+
* Releases only Flycast addresses independently validated for this network.
|
|
821
|
+
* Metadata identifies candidates; it never authorizes a cross-network DELETE.
|
|
781
822
|
*/
|
|
782
|
-
async releasePrivateRoutes(routes) {
|
|
823
|
+
async releasePrivateRoutes(routes, signal, network) {
|
|
824
|
+
let released = true;
|
|
783
825
|
for (const [target, ip] of Object.entries(routes)) {
|
|
784
826
|
try {
|
|
785
|
-
await this.
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
827
|
+
if (!(await this.privateRouteMatchesNetwork(target, ip, network, signal))) {
|
|
828
|
+
released = false;
|
|
829
|
+
logger.warn('Kept a Fly private (Flycast) address with unvalidated network ownership', {
|
|
830
|
+
target,
|
|
831
|
+
ip,
|
|
832
|
+
network,
|
|
833
|
+
});
|
|
834
|
+
continue;
|
|
835
|
+
}
|
|
836
|
+
await this.requestForDestruction(`/apps/${target}/ip_assignments/${encodeURIComponent(ip)}`, { method: 'DELETE', nullOn: [404] }, signal);
|
|
789
837
|
logger.debug('Released a Fly private (Flycast) address', { target, ip });
|
|
790
838
|
}
|
|
791
839
|
catch (error) {
|
|
840
|
+
if (signal.aborted)
|
|
841
|
+
throw signal.reason;
|
|
842
|
+
released = false;
|
|
792
843
|
logger.warn('Failed to release a Fly private (Flycast) address — it will keep pointing at a 6PN ' +
|
|
793
844
|
'with no apps in it until it is reconciled with `fly ips list`', { target, ip, error });
|
|
794
845
|
}
|
|
795
846
|
}
|
|
847
|
+
return released;
|
|
796
848
|
}
|
|
797
849
|
/**
|
|
798
850
|
* Resolves the ports the egress network policy allows, or `undefined` when no
|
|
@@ -811,8 +863,8 @@ class FlyioSandboxProvider {
|
|
|
811
863
|
* about a rule with no ports, so accepting `[]` would ship a security
|
|
812
864
|
* control whose behaviour is a guess.
|
|
813
865
|
*/
|
|
814
|
-
egressAllowedPorts() {
|
|
815
|
-
return mergeEgressPorts(this.configuredEgressPorts(), this.privateServices());
|
|
866
|
+
egressAllowedPorts(createRegion) {
|
|
867
|
+
return mergeEgressPorts(this.configuredEgressPorts(), this.privateServices(createRegion));
|
|
816
868
|
}
|
|
817
869
|
/**
|
|
818
870
|
* Resolves ONLY the ports the operator configured, before the declared private
|
|
@@ -845,8 +897,8 @@ class FlyioSandboxProvider {
|
|
|
845
897
|
* @param app - The Fly app name.
|
|
846
898
|
* @returns Nothing; resolves once the policy is in place (or none was configured).
|
|
847
899
|
*/
|
|
848
|
-
async applyEgressPolicy(app) {
|
|
849
|
-
const ports = this.egressAllowedPorts();
|
|
900
|
+
async applyEgressPolicy(app, attempt, createRegion) {
|
|
901
|
+
const ports = this.egressAllowedPorts(createRegion);
|
|
850
902
|
if (!ports)
|
|
851
903
|
return;
|
|
852
904
|
const name = this.config.egressPolicyName ?? DEFAULT_EGRESS_POLICY_NAME;
|
|
@@ -878,7 +930,18 @@ class FlyioSandboxProvider {
|
|
|
878
930
|
selector: { all: true },
|
|
879
931
|
rules: [{ action: 'allow', direction: 'egress', ports }],
|
|
880
932
|
};
|
|
881
|
-
|
|
933
|
+
if (attempt) {
|
|
934
|
+
await this.requestForAttempt(attempt, `/apps/${app}/network_policies`, {
|
|
935
|
+
method: 'POST',
|
|
936
|
+
body: policy,
|
|
937
|
+
});
|
|
938
|
+
}
|
|
939
|
+
else {
|
|
940
|
+
await this.client.request(`/apps/${app}/network_policies`, {
|
|
941
|
+
method: 'POST',
|
|
942
|
+
body: policy,
|
|
943
|
+
});
|
|
944
|
+
}
|
|
882
945
|
logger.info('Applied Fly egress network policy', {
|
|
883
946
|
app,
|
|
884
947
|
name,
|
|
@@ -956,7 +1019,7 @@ class FlyioSandboxProvider {
|
|
|
956
1019
|
* rather than failing the boot.
|
|
957
1020
|
* @param app - The Fly app name.
|
|
958
1021
|
*/
|
|
959
|
-
async assignSharedIpv4(app) {
|
|
1022
|
+
async assignSharedIpv4(app, attempt) {
|
|
960
1023
|
try {
|
|
961
1024
|
// ONLY `type` — verified against the live API 2026-08-09. Both `region`
|
|
962
1025
|
// ("region selection is not supported for this IP type", 400) and
|
|
@@ -964,37 +1027,51 @@ class FlyioSandboxProvider {
|
|
|
964
1027
|
// sandbox app WITHOUT a public IP, so its preview URL never routed and the
|
|
965
1028
|
// IDE hung on "Loading preview…" forever. The org derives from the app and
|
|
966
1029
|
// a shared v4 is global (no region).
|
|
967
|
-
|
|
1030
|
+
const request = {
|
|
968
1031
|
method: 'POST',
|
|
969
|
-
body: {
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
1032
|
+
body: { type: this.config.ipAssignmentType ?? 'shared_v4' },
|
|
1033
|
+
};
|
|
1034
|
+
if (attempt)
|
|
1035
|
+
await this.requestForAttempt(attempt, `/apps/${app}/ip_assignments`, request);
|
|
1036
|
+
else
|
|
1037
|
+
await this.client.request(`/apps/${app}/ip_assignments`, request);
|
|
973
1038
|
}
|
|
974
1039
|
catch (error) {
|
|
1040
|
+
if (attempt?.signal.aborted)
|
|
1041
|
+
throw attempt.signal.reason;
|
|
1042
|
+
if (attempt) {
|
|
1043
|
+
await this.config.createAttemptJournal?.assertGeneration(attempt.attemptId, attempt.generation);
|
|
1044
|
+
}
|
|
975
1045
|
logger.warn('Failed to assign a shared IPv4 to the Fly sandbox app — the public preview URL will ' +
|
|
976
1046
|
'not resolve until an address is allocated', { app, error });
|
|
977
1047
|
}
|
|
978
1048
|
}
|
|
979
1049
|
/**
|
|
980
|
-
*
|
|
981
|
-
*
|
|
982
|
-
*
|
|
983
|
-
* rejects anything else with `400 "name only allows lowercase alphanumeric
|
|
984
|
-
* characters and underscores with at most 30 characters"`. Callers pass names
|
|
985
|
-
* like `mol-<uuid>` (40 chars, hyphens): lowercase, map every other character
|
|
986
|
-
* to `_`, and when the result is too long, keep a readable prefix plus a
|
|
987
|
-
* deterministic hash suffix so the name stays stable (`findVolume` re-derives
|
|
988
|
-
* the same name on the idempotent path) and unique.
|
|
989
|
-
* @param name - The caller's volume name.
|
|
990
|
-
* @returns A sanitized Fly volume name, `[a-z0-9_]` and <= 30 chars.
|
|
1050
|
+
* Return the readable, sanitized identity shared by attempts for one caller label.
|
|
1051
|
+
* @param name - The caller's volume label.
|
|
1052
|
+
* @returns The `[a-z0-9_]` stem used by {@link volumeName}.
|
|
991
1053
|
*/
|
|
992
|
-
|
|
1054
|
+
volumeNameStem(name) {
|
|
1055
|
+
const sanitized = name.toLowerCase().replace(/[^a-z0-9_]/g, '_');
|
|
1056
|
+
return `${sanitized.slice(0, 17)}_`;
|
|
1057
|
+
}
|
|
1058
|
+
/**
|
|
1059
|
+
* Give one create attempt a unique Fly volume name (Fly permits duplicate names).
|
|
1060
|
+
* @param name - The caller's volume label.
|
|
1061
|
+
* @param attemptId - The Machine create attempt's identity.
|
|
1062
|
+
* @returns A sanitized name of at most 30 characters, including the full attempt's digest.
|
|
1063
|
+
*/
|
|
1064
|
+
volumeName(name, attemptId) {
|
|
1065
|
+
const digest = createHash('sha256').update(`${name}\0${attemptId}`).digest('hex').slice(0, 12);
|
|
1066
|
+
return `${this.volumeNameStem(name)}${digest}`;
|
|
1067
|
+
}
|
|
1068
|
+
/** The stable pre-attempt name under which durable workspaces were created. */
|
|
1069
|
+
workspaceVolumeName(name) {
|
|
993
1070
|
const sanitized = name.toLowerCase().replace(/[^a-z0-9_]/g, '_');
|
|
994
1071
|
if (sanitized.length <= 30)
|
|
995
1072
|
return sanitized;
|
|
996
|
-
const
|
|
997
|
-
return `${sanitized.slice(0, 17)}_${
|
|
1073
|
+
const digest = createHash('sha256').update(name).digest('hex').slice(0, 12);
|
|
1074
|
+
return `${sanitized.slice(0, 17)}_${digest}`;
|
|
998
1075
|
}
|
|
999
1076
|
/**
|
|
1000
1077
|
* Idempotently ensures a volume exists in the app and returns its id.
|
|
@@ -1004,28 +1081,138 @@ class FlyioSandboxProvider {
|
|
|
1004
1081
|
* size are all known — rather than through the core's optional
|
|
1005
1082
|
* `createVolume(name)`; see the module `@remarks`.
|
|
1006
1083
|
* @param app - The Fly app name.
|
|
1007
|
-
* @param
|
|
1084
|
+
* @param flyName - This create attempt's unique, already-sanitized volume name.
|
|
1008
1085
|
* @param sizeGB - Size to create the volume with, in whole GB.
|
|
1009
|
-
* @
|
|
1086
|
+
* @param compute - The exact guest sent with the Machine create.
|
|
1087
|
+
* @returns The Fly volume id and whether this call created the volume.
|
|
1010
1088
|
*/
|
|
1011
|
-
async ensureVolume(app, name, sizeGB) {
|
|
1012
|
-
const
|
|
1013
|
-
const
|
|
1014
|
-
if (
|
|
1015
|
-
|
|
1016
|
-
|
|
1089
|
+
async ensureVolume(app, projectId, name, attempt, sizeGB, compute, region) {
|
|
1090
|
+
const journal = this.config.createAttemptJournal;
|
|
1091
|
+
const bound = await journal?.findWorkspace(projectId, name);
|
|
1092
|
+
if (bound?.id) {
|
|
1093
|
+
const volumes = await this.client.request(`/apps/${app}/volumes`, {
|
|
1094
|
+
signal: attempt.signal,
|
|
1095
|
+
});
|
|
1096
|
+
const existing = (volumes ?? []).find((volume) => volume.id === bound.id && volume.state !== 'destroyed' && volume.state !== 'destroying');
|
|
1097
|
+
if (!existing) {
|
|
1098
|
+
throw new Error(`Durable workspace volume "${bound.id}" cannot be read; refusing to replace it`);
|
|
1099
|
+
}
|
|
1100
|
+
await journal?.recordVolume(attempt.attemptId, {
|
|
1101
|
+
...bound,
|
|
1102
|
+
id: existing.id,
|
|
1103
|
+
logicalName: name,
|
|
1104
|
+
workspace: true,
|
|
1105
|
+
resolved: true,
|
|
1106
|
+
createdByAttemptId: undefined,
|
|
1107
|
+
adoptedByAttemptId: attempt.attemptId,
|
|
1108
|
+
});
|
|
1109
|
+
return { id: existing.id, name: bound.name, workspace: true };
|
|
1110
|
+
}
|
|
1111
|
+
// Pre-attempt releases used a deterministic name. Adopt that legacy volume
|
|
1112
|
+
// by ID once; all later recreation uses the durable binding above.
|
|
1113
|
+
const durableName = this.workspaceVolumeName(name);
|
|
1114
|
+
const existing = await this.findVolume(app, durableName, attempt.signal);
|
|
1115
|
+
if (existing) {
|
|
1116
|
+
await journal?.recordVolume(attempt.attemptId, {
|
|
1117
|
+
id: existing.id,
|
|
1118
|
+
name: durableName,
|
|
1119
|
+
logicalName: name,
|
|
1120
|
+
workspace: true,
|
|
1121
|
+
resolved: true,
|
|
1122
|
+
adoptedByAttemptId: attempt.attemptId,
|
|
1123
|
+
});
|
|
1124
|
+
return { id: existing.id, name: durableName, workspace: true };
|
|
1125
|
+
}
|
|
1126
|
+
const attemptName = this.volumeName(name, attempt.attemptId);
|
|
1127
|
+
await journal?.recordVolume(attempt.attemptId, {
|
|
1128
|
+
name: attemptName,
|
|
1129
|
+
logicalName: name,
|
|
1130
|
+
workspace: true,
|
|
1131
|
+
resolved: false,
|
|
1132
|
+
createdByAttemptId: attempt.attemptId,
|
|
1133
|
+
});
|
|
1134
|
+
try {
|
|
1135
|
+
const created = await this.createFreshVolume(app, attemptName, attempt, sizeGB, compute, region);
|
|
1136
|
+
await journal?.recordVolume(attempt.attemptId, {
|
|
1137
|
+
id: created.id,
|
|
1138
|
+
name: attemptName,
|
|
1139
|
+
logicalName: name,
|
|
1140
|
+
workspace: false,
|
|
1141
|
+
resolved: true,
|
|
1142
|
+
createdByAttemptId: created.accepted ? attempt.attemptId : undefined,
|
|
1143
|
+
adoptedByAttemptId: created.accepted ? undefined : attempt.attemptId,
|
|
1144
|
+
});
|
|
1145
|
+
return { id: created.id, name: attemptName, workspace: false };
|
|
1146
|
+
}
|
|
1147
|
+
catch (error) {
|
|
1148
|
+
if (error instanceof FlyApiError &&
|
|
1149
|
+
error.method.toUpperCase() === 'POST' &&
|
|
1150
|
+
error.path === `/apps/${app}/volumes` &&
|
|
1151
|
+
error.status >= 400 &&
|
|
1152
|
+
error.status < 500 &&
|
|
1153
|
+
![409, 422].includes(error.status)) {
|
|
1154
|
+
// A conflict/validation response can describe a write whose earlier
|
|
1155
|
+
// response was lost, so those outcomes stay unresolved. The 429 replay
|
|
1156
|
+
// contract treats rate-limit rejection as definite; other non-conflict
|
|
1157
|
+
// 4xx responses reject this unique attempt name before creating it.
|
|
1158
|
+
await journal?.recordVolume(attempt.attemptId, {
|
|
1159
|
+
name: attemptName,
|
|
1160
|
+
logicalName: name,
|
|
1161
|
+
workspace: false,
|
|
1162
|
+
resolved: true,
|
|
1163
|
+
});
|
|
1164
|
+
}
|
|
1165
|
+
throw error;
|
|
1166
|
+
}
|
|
1167
|
+
}
|
|
1168
|
+
/**
|
|
1169
|
+
* Creates a volume with the expected Machine's guest as its placement hint.
|
|
1170
|
+
* A 429 is safe to replay directly. After a transport failure or 5xx, bounded
|
|
1171
|
+
* lookups reconcile the attempt's unique name; if it stays invisible the create
|
|
1172
|
+
* fails rather than replaying an ambiguous POST that could duplicate the volume.
|
|
1173
|
+
* @param app - The Fly app name.
|
|
1174
|
+
* @param flyName - This create attempt's unique, already-sanitized volume name.
|
|
1175
|
+
* @param sizeGB - Size to create the volume with, in whole GB.
|
|
1176
|
+
* @param compute - The exact guest sent with the Machine create.
|
|
1177
|
+
* @returns The new Fly volume id.
|
|
1178
|
+
*/
|
|
1179
|
+
async createFreshVolume(app, flyName, attempt, sizeGB, compute, region) {
|
|
1180
|
+
let adopted = false;
|
|
1181
|
+
const created = await this.requestForAttempt(attempt, `/apps/${app}/volumes`, {
|
|
1017
1182
|
method: 'POST',
|
|
1018
1183
|
body: {
|
|
1019
1184
|
name: flyName,
|
|
1020
|
-
region
|
|
1185
|
+
region,
|
|
1021
1186
|
size_gb: sizeGB,
|
|
1022
1187
|
encrypted: true,
|
|
1188
|
+
compute,
|
|
1189
|
+
},
|
|
1190
|
+
reconcile: async (error) => {
|
|
1191
|
+
// A 429 is Fly's confirmation that this request was rejected. Every
|
|
1192
|
+
// other retryable outcome is ambiguous: never replay it until a few
|
|
1193
|
+
// name lookups have had a chance to find the volume Fly may have made.
|
|
1194
|
+
if (error.status !== 0 && error.status < 500)
|
|
1195
|
+
return undefined;
|
|
1196
|
+
for (let lookup = 1; lookup <= CREATE_RECONCILE_ATTEMPTS; lookup++) {
|
|
1197
|
+
const existing = await this.findVolume(app, flyName, attempt.signal);
|
|
1198
|
+
if (existing) {
|
|
1199
|
+
adopted = true;
|
|
1200
|
+
return existing;
|
|
1201
|
+
}
|
|
1202
|
+
if (lookup < CREATE_RECONCILE_ATTEMPTS) {
|
|
1203
|
+
await this.client.sleep(retryDelayMs(lookup));
|
|
1204
|
+
}
|
|
1205
|
+
}
|
|
1206
|
+
// Fly permits duplicate volume names. Negative reads therefore cannot
|
|
1207
|
+
// authorize another POST: stop with the outcome unknown and let the
|
|
1208
|
+
// owning app's durable reconciler reclaim anything the write made.
|
|
1209
|
+
throw new Error(`Fly volume create "${flyName}" has an unknown outcome; refusing to replay it`, { cause: error });
|
|
1023
1210
|
},
|
|
1024
1211
|
});
|
|
1025
1212
|
if (!created?.id) {
|
|
1026
1213
|
throw new Error(t('codeSandbox.flyio.error.volumeCreateFailed', { name: flyName }, { defaultValue: `Fly volume "${flyName}" was created but the API returned no id.` }));
|
|
1027
1214
|
}
|
|
1028
|
-
return created.id;
|
|
1215
|
+
return { id: created.id, accepted: !adopted };
|
|
1029
1216
|
}
|
|
1030
1217
|
/**
|
|
1031
1218
|
* Finds a volume in an app by name, ignoring ones already being destroyed.
|
|
@@ -1033,9 +1220,10 @@ class FlyioSandboxProvider {
|
|
|
1033
1220
|
* @param flyName - The sanitized Fly volume name.
|
|
1034
1221
|
* @returns The matching volume, or `null`.
|
|
1035
1222
|
*/
|
|
1036
|
-
async findVolume(app, flyName) {
|
|
1223
|
+
async findVolume(app, flyName, signal) {
|
|
1037
1224
|
const volumes = await this.client.request(`/apps/${app}/volumes`, {
|
|
1038
1225
|
nullOn: [404],
|
|
1226
|
+
signal,
|
|
1039
1227
|
});
|
|
1040
1228
|
if (!Array.isArray(volumes))
|
|
1041
1229
|
return null;
|
|
@@ -1063,7 +1251,21 @@ class FlyioSandboxProvider {
|
|
|
1063
1251
|
* @param config - Project id, image, env, volume name, template, labels and resource limits.
|
|
1064
1252
|
* @returns A `Sandbox` bound to the new Machine.
|
|
1065
1253
|
*/
|
|
1066
|
-
|
|
1254
|
+
/**
|
|
1255
|
+
* Return the collision-free app name for a complete caller/project identity.
|
|
1256
|
+
* @param projectId - The complete project or per-deploy identity.
|
|
1257
|
+
* @returns The DNS-safe, at-most-63-character app name.
|
|
1258
|
+
*/
|
|
1259
|
+
appNameForProject(projectId) {
|
|
1260
|
+
return appNameForProject(this.appPrefix(), projectId);
|
|
1261
|
+
}
|
|
1262
|
+
/**
|
|
1263
|
+
* Create, reconcile and boot one uniquely identified Fly Machine attempt.
|
|
1264
|
+
* Automatic cleanup deletes only a volume or exact Machine ID with this attempt's accepted-create provenance. Apps always remain KEEP for deliberate retirement; adopted, inherited, listed, older-writer and ambiguous resources are also KEPT.
|
|
1265
|
+
* @param config - The sandbox image, project, storage and resource configuration.
|
|
1266
|
+
* @returns The running sandbox.
|
|
1267
|
+
*/
|
|
1268
|
+
async create(config, signal = new AbortController().signal) {
|
|
1067
1269
|
// Resolved BEFORE anything is provisioned, so a bad template id costs no app,
|
|
1068
1270
|
// no volume and no Machine.
|
|
1069
1271
|
let manifest = null;
|
|
@@ -1083,49 +1285,60 @@ class FlyioSandboxProvider {
|
|
|
1083
1285
|
// is checked too — the host is unroutable however the value arrives, and
|
|
1084
1286
|
// that env is exactly where a control plane puts the values it refuses to
|
|
1085
1287
|
// leave in a Machine config.
|
|
1086
|
-
|
|
1288
|
+
const createRegion = this.region(config.region);
|
|
1289
|
+
assertPrivateRoutesForEnv({ ...config.env, ...config.selfDeliveredEnv }, this.privateServices(createRegion));
|
|
1087
1290
|
const app = this.resolveApp(config.projectId);
|
|
1088
|
-
const
|
|
1291
|
+
const createAttemptId = randomUUID();
|
|
1292
|
+
const journal = this.config.createAttemptJournal;
|
|
1293
|
+
// The SQL journal stays closed until pre-journal/provenance-ignoring writers
|
|
1294
|
+
// have drained. No durable row or Fly resource is created while it is closed.
|
|
1295
|
+
await journal?.assertCreateAllowed();
|
|
1296
|
+
await journal?.begin({
|
|
1297
|
+
attemptId: createAttemptId,
|
|
1298
|
+
projectId: config.projectId,
|
|
1299
|
+
ownerProjectId: String(config.labels?.['molecule.production'] ?? config.projectId),
|
|
1300
|
+
appName: app,
|
|
1301
|
+
});
|
|
1302
|
+
const attempt = { attemptId: createAttemptId, generation: 1, signal };
|
|
1089
1303
|
try {
|
|
1090
|
-
|
|
1304
|
+
const { routes: privateRoutes } = await this.ensureApp(app, {
|
|
1305
|
+
attempt,
|
|
1306
|
+
createRegion,
|
|
1307
|
+
});
|
|
1308
|
+
const sandbox = await this.createInApp(app, privateRoutes, config, manifest, attempt);
|
|
1309
|
+
if (signal.aborted)
|
|
1310
|
+
throw signal.reason;
|
|
1311
|
+
await journal?.assertGeneration(attempt.attemptId, attempt.generation);
|
|
1312
|
+
await journal?.markState(createAttemptId, 'serving', sandbox.id);
|
|
1313
|
+
return sandbox;
|
|
1091
1314
|
}
|
|
1092
1315
|
catch (error) {
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
// Machine that would have carried the metadata was never created, and
|
|
1103
|
-
// Fly's listing cannot say which network an address belongs to. Take
|
|
1104
|
-
// back exactly this call's own allocations — never the routes an
|
|
1105
|
-
// earlier boot recorded, whose Machines may still be running.
|
|
1106
|
-
await this.releasePrivateRoutes(allocated);
|
|
1107
|
-
}
|
|
1316
|
+
// Create has NO deletion path. The durable attempt is settled by the
|
|
1317
|
+
// separate reconciler after this failure (or after its abandonment window).
|
|
1318
|
+
await journal?.markState(createAttemptId, 'failed').catch((journalError) => {
|
|
1319
|
+
logger.warn('Could not mark a Fly create attempt failed', {
|
|
1320
|
+
app,
|
|
1321
|
+
createAttemptId,
|
|
1322
|
+
error: journalError,
|
|
1323
|
+
});
|
|
1324
|
+
});
|
|
1108
1325
|
throw error;
|
|
1109
1326
|
}
|
|
1110
1327
|
}
|
|
1111
1328
|
/**
|
|
1112
|
-
* The part of {@link create} that
|
|
1113
|
-
* Machines, provision the volume, create the Machine and wait for it.
|
|
1329
|
+
* The part of {@link create} that provisions the volume, creates the Machine and waits for it.
|
|
1114
1330
|
* @param app - The app.
|
|
1115
1331
|
* @param privateRoutes - Flycast addresses allocated for the app.
|
|
1116
1332
|
* @param config - The sandbox config.
|
|
1117
1333
|
* @param manifest - The template manifest, when creating from a template.
|
|
1118
1334
|
* @returns The running sandbox.
|
|
1119
1335
|
*/
|
|
1120
|
-
async createInApp(app, privateRoutes, config, manifest) {
|
|
1121
|
-
|
|
1122
|
-
// provisioning — otherwise it still holds the app's single-attach volume and
|
|
1123
|
-
// `POST /machines` fails with `412 volume already claimed`. Makes create
|
|
1124
|
-
// idempotent so a retried queue job (or a user re-clicking Start) recovers.
|
|
1125
|
-
await this.destroyStaleMachines(app);
|
|
1336
|
+
async createInApp(app, privateRoutes, config, manifest, attempt) {
|
|
1337
|
+
const createAttemptId = attempt.attemptId;
|
|
1126
1338
|
const metadata = {
|
|
1127
1339
|
[`${this.metadataPrefix}.projectId`]: config.projectId,
|
|
1128
1340
|
[`${this.metadataPrefix}.managed`]: 'true',
|
|
1341
|
+
[`${this.metadataPrefix}.createAttempt`]: createAttemptId,
|
|
1129
1342
|
// Recorded so `destroy()` can release exactly the addresses this project
|
|
1130
1343
|
// allocated. Fly's IP listing does not report which network an address
|
|
1131
1344
|
// belongs to, so nothing else can reconstruct this.
|
|
@@ -1171,34 +1384,112 @@ class FlyioSandboxProvider {
|
|
|
1171
1384
|
// the image's idle command and serving nothing.
|
|
1172
1385
|
if (config.command?.length)
|
|
1173
1386
|
machineConfig.init = { exec: [...config.command] };
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1387
|
+
const volumeName = config.volumeName;
|
|
1388
|
+
if (volumeName) {
|
|
1389
|
+
metadata[`${this.metadataPrefix}.volumeName`] = volumeName;
|
|
1390
|
+
const volumeSizeGB = config.resources?.diskMB
|
|
1177
1391
|
? Math.max(1, Math.ceil(config.resources.diskMB / 1024))
|
|
1178
1392
|
: (this.config.defaultVolumeGB ?? DEFAULT_VOLUME_GB);
|
|
1179
|
-
const
|
|
1393
|
+
const volume = await this.ensureVolume(app, config.projectId, volumeName, attempt, volumeSizeGB, machineConfig.guest ?? {}, this.region(config.region));
|
|
1180
1394
|
// The caller may scope the volume to a subtree (e.g. /workspace/<appDir>)
|
|
1181
1395
|
// so regenerable tooling (node_modules) stays on the fast image-backed
|
|
1182
1396
|
// rootfs and only durable state pays the volume's small-file IO cost.
|
|
1183
|
-
machineConfig.mounts = [{ volume:
|
|
1397
|
+
machineConfig.mounts = [{ volume: volume.id, path: config.volumeMountPath ?? WORKSPACE_PATH }];
|
|
1184
1398
|
}
|
|
1185
1399
|
const service = this.previewService(config.idleAction);
|
|
1186
1400
|
if (service)
|
|
1187
1401
|
machineConfig.services = [service];
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1402
|
+
// The POST intent must survive a crash before dispatch. A NULL outcome on
|
|
1403
|
+
// an older row is unknown too; only a new not-started/rejected row proves
|
|
1404
|
+
// that no accepted Machine identity can be hidden by a later empty list.
|
|
1405
|
+
await this.config.createAttemptJournal?.recordMachineCreateOutcome(createAttemptId, attempt.generation, 'pending');
|
|
1406
|
+
let machine;
|
|
1407
|
+
let machineCreateResponseLost = false;
|
|
1408
|
+
try {
|
|
1409
|
+
machine = await this.requestForAttempt(attempt, `/apps/${app}/machines`, {
|
|
1410
|
+
method: 'POST',
|
|
1411
|
+
body: {
|
|
1412
|
+
name: createAttemptId,
|
|
1413
|
+
region: this.region(config.region),
|
|
1414
|
+
config: machineConfig,
|
|
1415
|
+
},
|
|
1416
|
+
// A just-created app propagates to the Machines API asynchronously, so this
|
|
1417
|
+
// POST can 404 "app not found" for a few seconds. Retry the 404 (safe: no
|
|
1418
|
+
// Machine is created when the app is not found) across the client's backoff.
|
|
1419
|
+
attempts: MACHINE_CREATE_ATTEMPTS,
|
|
1420
|
+
retryStatuses: MACHINE_CREATE_RETRY_STATUSES,
|
|
1421
|
+
// A 412 is transient only for the specific released-volume-claim lag.
|
|
1422
|
+
// Placement capacity must reach the caller's bounded retry at once,
|
|
1423
|
+
// with the volume preserved; every other 412 remains a real rejection.
|
|
1424
|
+
retryWhen: (error) => {
|
|
1425
|
+
if (error.status === 400)
|
|
1426
|
+
return /volume not found/i.test(error.body);
|
|
1427
|
+
if (error.status !== 412)
|
|
1428
|
+
return true;
|
|
1429
|
+
return (/volume already claimed by machine/i.test(error.body) ||
|
|
1430
|
+
/volume_placement_capacity/i.test(error.body));
|
|
1431
|
+
},
|
|
1432
|
+
reconcile: async (error) => {
|
|
1433
|
+
// Rate limiting and explicit precondition rejections did not create a
|
|
1434
|
+
// Machine, so their normal bounded replay is safe. A transport/5xx
|
|
1435
|
+
// result is ambiguous: reconcile ONLY this attempt's recorded identity
|
|
1436
|
+
// and never issue another create when the reads cannot find it.
|
|
1437
|
+
if (error.status === 429 || (error.status >= 400 && error.status < 500)) {
|
|
1438
|
+
return undefined;
|
|
1439
|
+
}
|
|
1440
|
+
// A Machine recovered by listing after a lost response is adoption
|
|
1441
|
+
// evidence, not an accepted POST response and never delete authority.
|
|
1442
|
+
machineCreateResponseLost = true;
|
|
1443
|
+
await this.config.createAttemptJournal?.recordMachineCreateOutcome(createAttemptId, attempt.generation, 'unknown');
|
|
1444
|
+
for (let lookup = 1; lookup <= CREATE_RECONCILE_ATTEMPTS; lookup++) {
|
|
1445
|
+
const machines = await this.client.request(`/apps/${app}/machines`, {
|
|
1446
|
+
nullOn: [404],
|
|
1447
|
+
signal: attempt.signal,
|
|
1448
|
+
});
|
|
1449
|
+
const existing = (machines ?? []).find((candidate) => candidate.name === createAttemptId ||
|
|
1450
|
+
candidate.config?.metadata?.[`${this.metadataPrefix}.createAttempt`] ===
|
|
1451
|
+
createAttemptId);
|
|
1452
|
+
if (existing) {
|
|
1453
|
+
await this.config.createAttemptJournal?.recordMachineObservation(createAttemptId, attempt.generation, existing.id);
|
|
1454
|
+
return existing;
|
|
1455
|
+
}
|
|
1456
|
+
if (lookup < CREATE_RECONCILE_ATTEMPTS) {
|
|
1457
|
+
await this.client.sleep(retryDelayMs(lookup));
|
|
1458
|
+
}
|
|
1459
|
+
}
|
|
1460
|
+
throw new Error(`Fly Machine create "${createAttemptId}" in app "${app}" has an unknown outcome; refusing to replay it`, { cause: error });
|
|
1461
|
+
},
|
|
1462
|
+
});
|
|
1463
|
+
}
|
|
1464
|
+
catch (error) {
|
|
1465
|
+
if (attempt.signal.aborted)
|
|
1466
|
+
throw attempt.signal.reason;
|
|
1467
|
+
if (error instanceof FlyApiError &&
|
|
1468
|
+
error.method.toUpperCase() === 'POST' &&
|
|
1469
|
+
error.path === `/apps/${app}/machines` &&
|
|
1470
|
+
error.status >= 400 &&
|
|
1471
|
+
error.status < 500) {
|
|
1472
|
+
try {
|
|
1473
|
+
await this.config.createAttemptJournal?.markMachineCreateFailed(createAttemptId, attempt.generation);
|
|
1474
|
+
}
|
|
1475
|
+
catch (journalError) {
|
|
1476
|
+
logger.warn('Could not record a definitive Fly Machine-create rejection', {
|
|
1477
|
+
app,
|
|
1478
|
+
createAttemptId,
|
|
1479
|
+
error: journalError,
|
|
1480
|
+
});
|
|
1481
|
+
}
|
|
1482
|
+
}
|
|
1483
|
+
throw error;
|
|
1484
|
+
}
|
|
1197
1485
|
if (!machine?.id) {
|
|
1198
1486
|
throw new Error(t('codeSandbox.flyio.error.createFailed', { app }, { defaultValue: `Fly Machine creation in app "${app}" returned no Machine id.` }));
|
|
1199
1487
|
}
|
|
1488
|
+
if (!machineCreateResponseLost) {
|
|
1489
|
+
await this.config.createAttemptJournal?.recordMachineProvenance(createAttemptId, attempt.generation, machine.id);
|
|
1490
|
+
}
|
|
1200
1491
|
if (manifest && config.templateId) {
|
|
1201
|
-
await this.restoreTemplate(app, machine.id, config.templateId, manifest);
|
|
1492
|
+
await this.restoreTemplate(app, machine.id, config.templateId, manifest, attempt);
|
|
1202
1493
|
// The restore waited for `started` and ran a command in it, so the state
|
|
1203
1494
|
// read back from the creation response is stale by definition.
|
|
1204
1495
|
return this.buildSandbox(app, machine.id, 'running');
|
|
@@ -1209,35 +1500,7 @@ class FlyioSandboxProvider {
|
|
|
1209
1500
|
// `412 failed_precondition: machine not running` until the Machine is up.
|
|
1210
1501
|
// Wait for `started` here so `create` always hands back a live sandbox, as
|
|
1211
1502
|
// the JSDoc promises — the template path already does this via restore.
|
|
1212
|
-
|
|
1213
|
-
await this.waitForStarted(app, machine.id);
|
|
1214
|
-
}
|
|
1215
|
-
catch (error) {
|
|
1216
|
-
// The boot failed after the Machine existed. This call created it and
|
|
1217
|
-
// handed it to NOBODY — the caller got a rejection, not a handle — so
|
|
1218
|
-
// leaving it behind strands state nothing else ever cleans up: a Machine
|
|
1219
|
-
// that reached `started` bills compute until it is reaped, and one that
|
|
1220
|
-
// did not keeps the app's single-attach volume claimed (the next
|
|
1221
|
-
// create's stale-Machine sweep only runs if this project is retried).
|
|
1222
|
-
// The created-app failure path below cannot do this either: it deletes
|
|
1223
|
-
// the APP, and `discardCreatedApp` deliberately has no Machine-delete
|
|
1224
|
-
// step — an app delete that runs under a live Machine either fails or
|
|
1225
|
-
// hides the Machine. Destroy the Machine first (the same order
|
|
1226
|
-
// `restoreTemplate`'s failure path and `destroy()` use), then rethrow
|
|
1227
|
-
// the boot failure. Best-effort: the boot failure is what the caller
|
|
1228
|
-
// must see; a failed cleanup is logged like every other cleanup here.
|
|
1229
|
-
try {
|
|
1230
|
-
await this.client.request(`/apps/${app}/machines/${machine.id}?force=true`, {
|
|
1231
|
-
method: 'DELETE',
|
|
1232
|
-
nullOn: [404],
|
|
1233
|
-
});
|
|
1234
|
-
}
|
|
1235
|
-
catch (cleanupError) {
|
|
1236
|
-
logger.warn('Failed to destroy the Fly Machine whose boot did not reach started — it will keep ' +
|
|
1237
|
-
'billing (and holding its volume) until it is reaped', { app, machineId: machine.id, error: cleanupError });
|
|
1238
|
-
}
|
|
1239
|
-
throw error;
|
|
1240
|
-
}
|
|
1503
|
+
await this.waitForStarted(app, machine.id, attempt);
|
|
1241
1504
|
return this.buildSandbox(app, machine.id, 'running');
|
|
1242
1505
|
}
|
|
1243
1506
|
/**
|
|
@@ -1476,8 +1739,10 @@ class FlyioSandboxProvider {
|
|
|
1476
1739
|
'TRUNCATED, so some sandboxes would be missing from the result.',
|
|
1477
1740
|
}));
|
|
1478
1741
|
}
|
|
1479
|
-
const
|
|
1480
|
-
const owned = apps
|
|
1742
|
+
const namespace = appNameNamespace(this.appPrefix());
|
|
1743
|
+
const owned = apps
|
|
1744
|
+
.map((app) => app.name)
|
|
1745
|
+
.filter((name) => namespace.searchPrefixes.some((prefix) => name?.startsWith(prefix)));
|
|
1481
1746
|
if (owned.length > FIND_MAX_APPS) {
|
|
1482
1747
|
throw new Error(t('codeSandbox.flyio.error.findTooManyApps', { count: String(owned.length), max: String(FIND_MAX_APPS) }, {
|
|
1483
1748
|
defaultValue: `Refusing to enumerate ${owned.length} Fly sandbox apps (cap ${FIND_MAX_APPS}): ` +
|
|
@@ -1487,6 +1752,834 @@ class FlyioSandboxProvider {
|
|
|
1487
1752
|
}
|
|
1488
1753
|
return owned;
|
|
1489
1754
|
}
|
|
1755
|
+
/** Read an existing app's live state; a list 404 after the app read is an error, not absence. */
|
|
1756
|
+
async readAttemptAppState(app, signal) {
|
|
1757
|
+
const [machines, volumes] = await Promise.all([
|
|
1758
|
+
this.requestForDestruction(`/apps/${app}/machines`, { nullOn: [404] }, signal),
|
|
1759
|
+
this.requestForDestruction(`/apps/${app}/volumes`, { nullOn: [404] }, signal),
|
|
1760
|
+
]);
|
|
1761
|
+
if (Array.isArray(machines) && Array.isArray(volumes)) {
|
|
1762
|
+
return {
|
|
1763
|
+
appGone: false,
|
|
1764
|
+
machines,
|
|
1765
|
+
volumes: volumes.filter((volume) => volume.state !== 'destroyed' && volume.state !== 'destroying'),
|
|
1766
|
+
};
|
|
1767
|
+
}
|
|
1768
|
+
// A Machines/Volumes-list 404 is not proof that a newly-created app is
|
|
1769
|
+
// gone. Confirm the app itself; if it exists, the list read was transient
|
|
1770
|
+
// and responsibility must remain durable.
|
|
1771
|
+
const existing = await this.requestForDestruction(`/apps/${app}`, { nullOn: [404] }, signal);
|
|
1772
|
+
if (existing)
|
|
1773
|
+
throw new Error(`Fly app "${app}" exists but its live-state listing failed`);
|
|
1774
|
+
return { appGone: true, machines: [], volumes: [] };
|
|
1775
|
+
}
|
|
1776
|
+
/** Prove a definitively rejected Machine create has no live consumer before retiring it. */
|
|
1777
|
+
async assertFailedMachineSweepable(intent, signal) {
|
|
1778
|
+
if ((intent.appCreatedByAttemptId === intent.attemptId && intent.appCreateResolved !== true) ||
|
|
1779
|
+
intent.volumes.some((volume) => volume.resolved !== true) ||
|
|
1780
|
+
intent.addresses.some((address) => !address.resolved)) {
|
|
1781
|
+
throw new Error(`Fly create attempt "${intent.attemptId}" still has unresolved ownership`);
|
|
1782
|
+
}
|
|
1783
|
+
const live = await this.readAttemptAppState(intent.appName, signal);
|
|
1784
|
+
if (signal.aborted)
|
|
1785
|
+
throw signal.reason;
|
|
1786
|
+
// Siblings matter only through an actual resource reference below, or at
|
|
1787
|
+
// the later app-deletion boundary. Their mere existence must not strand a
|
|
1788
|
+
// detached, proven-created volume in a shared app.
|
|
1789
|
+
if (live.appGone)
|
|
1790
|
+
return;
|
|
1791
|
+
if (this.appPerProject() && live.machines.length) {
|
|
1792
|
+
throw new Error(`Fly app "${intent.appName}" still has Machines; failed-create sweep is KEEP`);
|
|
1793
|
+
}
|
|
1794
|
+
const volumeIds = new Set(intent.volumes.map((volume) => volume.id).filter((id) => Boolean(id)));
|
|
1795
|
+
for (const machine of live.machines) {
|
|
1796
|
+
if (machine.config?.mounts?.some((mount) => mount.volume && volumeIds.has(mount.volume))) {
|
|
1797
|
+
throw new Error(`Fly Machine "${machine.id}" still references a failed-create volume; sweep is KEEP`);
|
|
1798
|
+
}
|
|
1799
|
+
}
|
|
1800
|
+
for (const volume of live.volumes) {
|
|
1801
|
+
if (volumeIds.has(volume.id) && volume.attached_machine_id) {
|
|
1802
|
+
throw new Error(`Fly volume "${volume.id}" is still attached to "${volume.attached_machine_id}"; sweep is KEEP`);
|
|
1803
|
+
}
|
|
1804
|
+
}
|
|
1805
|
+
}
|
|
1806
|
+
/** Exact durable Machine identity; unlike accepted provenance, sandboxId may be observational. */
|
|
1807
|
+
recordedMachineId(intent) {
|
|
1808
|
+
if (intent.machineId)
|
|
1809
|
+
return intent.machineId;
|
|
1810
|
+
const prefix = `${intent.appName}:`;
|
|
1811
|
+
if (!intent.sandboxId?.startsWith(prefix))
|
|
1812
|
+
return undefined;
|
|
1813
|
+
return intent.sandboxId.slice(prefix.length) || undefined;
|
|
1814
|
+
}
|
|
1815
|
+
/** True when no exact durable Machine ID can settle a possibly-dispatched POST. */
|
|
1816
|
+
hasUnresolvedMachineIdentity(intent) {
|
|
1817
|
+
if (this.recordedMachineId(intent))
|
|
1818
|
+
return false;
|
|
1819
|
+
const legacyUnknown = intent.machineCreateOutcome === undefined &&
|
|
1820
|
+
intent.machineCreateFailed !== true &&
|
|
1821
|
+
!intent.machineId &&
|
|
1822
|
+
!intent.sandboxId &&
|
|
1823
|
+
!intent.machineAdopted;
|
|
1824
|
+
return (legacyUnknown ||
|
|
1825
|
+
intent.machineCreateOutcome === 'pending' ||
|
|
1826
|
+
intent.machineCreateOutcome === 'unknown' ||
|
|
1827
|
+
intent.machineCreateOutcome === 'accepted' ||
|
|
1828
|
+
Boolean(intent.machineId || intent.sandboxId || intent.machineAdopted));
|
|
1829
|
+
}
|
|
1830
|
+
/**
|
|
1831
|
+
* The sole forgetting guard. A listing is never an absence proof: every
|
|
1832
|
+
* durable Machine/volume ID needs this pass's authorized DELETE success/404
|
|
1833
|
+
* or its own exact GET 404. Unresolved identities remain visible as KEEP.
|
|
1834
|
+
*/
|
|
1835
|
+
async assertAuthoritativeForget(intent, signal, settled, app) {
|
|
1836
|
+
// A successful non-forced app DELETE (or exact app 404) is authoritative
|
|
1837
|
+
// for every resource contained by that app. External addresses still need
|
|
1838
|
+
// their own settled records below.
|
|
1839
|
+
const containerSettled = app.settled;
|
|
1840
|
+
const machineId = this.recordedMachineId(intent);
|
|
1841
|
+
if (!containerSettled && this.hasUnresolvedMachineIdentity(intent)) {
|
|
1842
|
+
throw new Error(`Fly Machine identity for attempt "${intent.attemptId}" is unresolved; forgetting is KEEP`);
|
|
1843
|
+
}
|
|
1844
|
+
if (!containerSettled && machineId && !settled.machineIds.has(machineId)) {
|
|
1845
|
+
const machine = await this.requestForDestruction(`/apps/${intent.appName}/machines/${machineId}`, { nullOn: [404] }, signal);
|
|
1846
|
+
if (machine) {
|
|
1847
|
+
throw new Error(`Fly Machine "${machineId}" for attempt "${intent.attemptId}" still exists; forgetting is KEEP`);
|
|
1848
|
+
}
|
|
1849
|
+
}
|
|
1850
|
+
for (const volume of containerSettled ? [] : intent.volumes) {
|
|
1851
|
+
if (volume.resolved !== true) {
|
|
1852
|
+
throw new Error(`Fly volume "${volume.id ?? volume.name}" for attempt "${intent.attemptId}" is unresolved; forgetting is KEEP`);
|
|
1853
|
+
}
|
|
1854
|
+
if (!volume.id || settled.volumeIds.has(volume.id))
|
|
1855
|
+
continue;
|
|
1856
|
+
const live = await this.requestForDestruction(`/apps/${intent.appName}/volumes/${volume.id}`, { nullOn: [404] }, signal);
|
|
1857
|
+
if (live) {
|
|
1858
|
+
throw new Error(`Fly volume "${volume.id}" for attempt "${intent.attemptId}" still exists; forgetting is KEEP`);
|
|
1859
|
+
}
|
|
1860
|
+
}
|
|
1861
|
+
if (intent.addresses.some((address) => !address.resolved || address.owned)) {
|
|
1862
|
+
throw new Error(`Fly addresses for attempt "${intent.attemptId}" are not settled; KEEP`);
|
|
1863
|
+
}
|
|
1864
|
+
if (app.required && !app.settled) {
|
|
1865
|
+
throw new Error(`Fly app "${intent.appName}" has no authoritative absence result; KEEP`);
|
|
1866
|
+
}
|
|
1867
|
+
}
|
|
1868
|
+
/**
|
|
1869
|
+
* The ONE automatic deletion/settlement function. Its four invariants are:
|
|
1870
|
+
* (a) it runs only under the owning attempt's lifecycle lock, and re-reads the
|
|
1871
|
+
* durable record and Fly live state inside that lock before EVERY delete;
|
|
1872
|
+
* (b) automatic deletion additionally requires this attempt's accepted-POST
|
|
1873
|
+
* provenance and exact resource ID; adoption is never deletion authority;
|
|
1874
|
+
* (c) unknown outcomes and transient absence retain responsibility. The one
|
|
1875
|
+
* forgetting boundary below requires an authoritative exact-ID result for
|
|
1876
|
+
* every resource: an exact GET 404, or an authorized DELETE success/404. A
|
|
1877
|
+
* listing, empty listing, NULL ID or "no recorded resources" proves no
|
|
1878
|
+
* absence; an unresolved identity keeps the row and reports KEEP. Durable
|
|
1879
|
+
* addresses are released before app deletion/forgetting; and (d) cancellation
|
|
1880
|
+
* plus the attempt's current generation fences every Fly mutation. A stale,
|
|
1881
|
+
* serving or unreadable attempt therefore cannot delete anything or be forgotten.
|
|
1882
|
+
*/
|
|
1883
|
+
async settleCreateAttempt(snapshot, signal, generation, options = {}) {
|
|
1884
|
+
const automatic = options.automatic ?? false;
|
|
1885
|
+
const retire = options.retire ?? false;
|
|
1886
|
+
const failedMachineSweep = options.failedMachineSweep ?? false;
|
|
1887
|
+
const deleteApp = options.deleteApp ?? true;
|
|
1888
|
+
const forget = options.forget ?? true;
|
|
1889
|
+
const appRequired = deleteApp && this.appPerProject();
|
|
1890
|
+
// Automatic reconciliation never deletes an app: a container DELETE can
|
|
1891
|
+
// cascade into any sibling's retained or not-yet-resolved user resources.
|
|
1892
|
+
// An existing app remains a durable KEEP for deliberate retirement.
|
|
1893
|
+
const mayDeleteApp = appRequired && !automatic;
|
|
1894
|
+
const journal = this.config.createAttemptJournal;
|
|
1895
|
+
if (!journal)
|
|
1896
|
+
throw new Error('A durable create-attempt journal is required for settlement');
|
|
1897
|
+
const fresh = async () => {
|
|
1898
|
+
if (signal.aborted)
|
|
1899
|
+
throw signal.reason;
|
|
1900
|
+
await journal.assertGeneration(snapshot.attemptId, generation);
|
|
1901
|
+
const current = await journal.get(snapshot.attemptId);
|
|
1902
|
+
if (signal.aborted)
|
|
1903
|
+
throw signal.reason;
|
|
1904
|
+
if (!current || (current.generation ?? 1) !== generation) {
|
|
1905
|
+
throw new Error(`Fly create attempt "${snapshot.attemptId}" was superseded`);
|
|
1906
|
+
}
|
|
1907
|
+
if (current.state !== 'failed' && current.state !== 'abandoned') {
|
|
1908
|
+
throw new Error(`Fly create attempt "${snapshot.attemptId}" is no longer settleable`);
|
|
1909
|
+
}
|
|
1910
|
+
if ((options.expectedAppName && current.appName !== options.expectedAppName) ||
|
|
1911
|
+
(options.expectedMachineId && this.recordedMachineId(current) !== options.expectedMachineId)) {
|
|
1912
|
+
throw new Error(`Fly create attempt "${snapshot.attemptId}" changed outside the caller-validated scope`);
|
|
1913
|
+
}
|
|
1914
|
+
return current;
|
|
1915
|
+
};
|
|
1916
|
+
const ownMachine = (machines, intent) => {
|
|
1917
|
+
// Once an exact ID is durable, a same-name Machine is potentially a
|
|
1918
|
+
// replacement and must never inherit that identity. machineId may be an
|
|
1919
|
+
// observation; only machineCreatedByAttemptId grants automatic authority.
|
|
1920
|
+
const recordedId = this.recordedMachineId(intent);
|
|
1921
|
+
if (recordedId)
|
|
1922
|
+
return machines.find((machine) => machine.id === recordedId);
|
|
1923
|
+
if (options.recordedResourcesOnly)
|
|
1924
|
+
return undefined;
|
|
1925
|
+
const direct = machines.find((machine) => machine.name === intent.attemptId ||
|
|
1926
|
+
machine.config?.metadata?.[`${this.metadataPrefix}.createAttempt`] === intent.attemptId);
|
|
1927
|
+
if (direct || !intent.attemptId.startsWith('legacy:') || !intent.sandboxId)
|
|
1928
|
+
return direct;
|
|
1929
|
+
const prefix = `${intent.appName}:`;
|
|
1930
|
+
if (!intent.sandboxId.startsWith(prefix))
|
|
1931
|
+
return undefined;
|
|
1932
|
+
const sandboxMachineId = intent.sandboxId.slice(prefix.length);
|
|
1933
|
+
return machines.find((machine) => machine.id === sandboxMachineId);
|
|
1934
|
+
};
|
|
1935
|
+
const adoptLegacyResources = async (machine) => {
|
|
1936
|
+
if (!intent.attemptId.startsWith('legacy:'))
|
|
1937
|
+
return;
|
|
1938
|
+
const metadata = machine.config?.metadata ?? {};
|
|
1939
|
+
const logicalName = metadata[`${this.metadataPrefix}.volumeName`];
|
|
1940
|
+
for (const mount of machine.config?.mounts ?? []) {
|
|
1941
|
+
if (!mount.volume || intent.volumes.some((volume) => volume.id === mount.volume))
|
|
1942
|
+
continue;
|
|
1943
|
+
const listed = live.volumes.find((volume) => volume.id === mount.volume);
|
|
1944
|
+
await journal.recordVolume(intent.attemptId, {
|
|
1945
|
+
id: mount.volume,
|
|
1946
|
+
name: listed?.name ?? mount.volume,
|
|
1947
|
+
...(logicalName ? { logicalName } : {}),
|
|
1948
|
+
workspace: false,
|
|
1949
|
+
resolved: true,
|
|
1950
|
+
adoptedByAttemptId: intent.attemptId,
|
|
1951
|
+
});
|
|
1952
|
+
}
|
|
1953
|
+
const routes = decodePrivateRoutes(metadata[`${this.metadataPrefix}.${PRIVATE_ROUTES_METADATA_SUFFIX}`]);
|
|
1954
|
+
const network = this.networkFor(intent.appName);
|
|
1955
|
+
for (const [targetApp, ip] of Object.entries(routes)) {
|
|
1956
|
+
// A durable exact binding for the intended network is authoritative.
|
|
1957
|
+
// Otherwise validate the IP independently; copied metadata is only a hint.
|
|
1958
|
+
if (intent.addresses.some((address) => address.targetApp === targetApp &&
|
|
1959
|
+
address.network === network &&
|
|
1960
|
+
address.ip === ip &&
|
|
1961
|
+
address.resolved)) {
|
|
1962
|
+
continue;
|
|
1963
|
+
}
|
|
1964
|
+
const validated = await this.privateRouteMatchesNetwork(targetApp, ip, network, signal);
|
|
1965
|
+
await journal.recordAddress(intent.attemptId, {
|
|
1966
|
+
targetApp,
|
|
1967
|
+
network: network ?? intent.appName,
|
|
1968
|
+
ip,
|
|
1969
|
+
owned: validated,
|
|
1970
|
+
resolved: validated,
|
|
1971
|
+
});
|
|
1972
|
+
}
|
|
1973
|
+
};
|
|
1974
|
+
const assertObservedMachineUnreferenced = async (intent, machineId) => {
|
|
1975
|
+
if (intent.machineCreatedByAttemptId === intent.attemptId)
|
|
1976
|
+
return;
|
|
1977
|
+
const siblings = await journal.forApp(intent.appName);
|
|
1978
|
+
if (signal.aborted)
|
|
1979
|
+
throw signal.reason;
|
|
1980
|
+
const conflict = siblings.find((candidate) => candidate.attemptId !== intent.attemptId &&
|
|
1981
|
+
candidate.state !== 'abandoned' &&
|
|
1982
|
+
this.recordedMachineId(candidate) === machineId);
|
|
1983
|
+
if (conflict) {
|
|
1984
|
+
throw new Error(`Fly Machine "${machineId}" is still referenced by ${conflict.state} attempt "${conflict.attemptId}"; observed retirement is KEEP`);
|
|
1985
|
+
}
|
|
1986
|
+
};
|
|
1987
|
+
const activeStates = [
|
|
1988
|
+
'creating',
|
|
1989
|
+
'starting',
|
|
1990
|
+
'started',
|
|
1991
|
+
'restarting',
|
|
1992
|
+
'updating',
|
|
1993
|
+
'replacing',
|
|
1994
|
+
'migrated',
|
|
1995
|
+
];
|
|
1996
|
+
const hasActiveSibling = async (intent) => {
|
|
1997
|
+
if (signal.aborted)
|
|
1998
|
+
throw signal.reason;
|
|
1999
|
+
const siblings = await journal.forApp(intent.appName);
|
|
2000
|
+
if (signal.aborted)
|
|
2001
|
+
throw signal.reason;
|
|
2002
|
+
return siblings.some((candidate) => {
|
|
2003
|
+
if (candidate.attemptId === intent.attemptId)
|
|
2004
|
+
return false;
|
|
2005
|
+
if (candidate.state === 'serving')
|
|
2006
|
+
return true;
|
|
2007
|
+
// Every expired sibling in this set is independently reclaimable and
|
|
2008
|
+
// will be retired under its own lock in this pass. The current app lock
|
|
2009
|
+
// fences it now; treating the stale row as live deadlocks both rows.
|
|
2010
|
+
return (candidate.state === 'creating' && !options.expiredCreatingIds?.has(candidate.attemptId));
|
|
2011
|
+
});
|
|
2012
|
+
};
|
|
2013
|
+
const readRecordedMachine = async (current) => {
|
|
2014
|
+
const id = this.recordedMachineId(current);
|
|
2015
|
+
if (!id)
|
|
2016
|
+
return undefined;
|
|
2017
|
+
return this.requestForDestruction(`/apps/${current.appName}/machines/${id}`, { nullOn: [404] }, signal);
|
|
2018
|
+
};
|
|
2019
|
+
const releasedMachineIds = new Set();
|
|
2020
|
+
let intent = await fresh();
|
|
2021
|
+
let live = await this.readAttemptAppState(intent.appName, signal);
|
|
2022
|
+
const observed = ownMachine(options.observedMachines ?? [], intent);
|
|
2023
|
+
if (observed) {
|
|
2024
|
+
// destroyApp read this metadata before waiting for the attempt lock. Save
|
|
2025
|
+
// that observation now; a later list omission cannot erase ownership.
|
|
2026
|
+
intent = await fresh();
|
|
2027
|
+
await adoptLegacyResources(observed);
|
|
2028
|
+
intent = await fresh();
|
|
2029
|
+
}
|
|
2030
|
+
// A list omission cannot erase a durable ID. Read that exact resource;
|
|
2031
|
+
// 403/5xx keeps the row, 404 is authoritative, and a live result follows
|
|
2032
|
+
// the same state/provenance checks as a listed Machine.
|
|
2033
|
+
const machine = ownMachine(live.machines, intent) ??
|
|
2034
|
+
(!forget || mayDeleteApp ? undefined : ((await readRecordedMachine(intent)) ?? undefined));
|
|
2035
|
+
if (machine && !this.recordedMachineId(intent)) {
|
|
2036
|
+
// A name/metadata match recovers an exact identity as observation only.
|
|
2037
|
+
// It deliberately does not gain automatic deletion provenance.
|
|
2038
|
+
await journal.recordMachineObservation(intent.attemptId, generation, machine.id);
|
|
2039
|
+
intent = await fresh();
|
|
2040
|
+
}
|
|
2041
|
+
if (automatic &&
|
|
2042
|
+
machine &&
|
|
2043
|
+
(intent.machineCreatedByAttemptId !== intent.attemptId || intent.machineId !== machine.id)) {
|
|
2044
|
+
throw new Error(`Fly Machine "${machine.id}" for attempt "${intent.attemptId}" has no matching accepted-POST provenance; KEEP`);
|
|
2045
|
+
}
|
|
2046
|
+
if (failedMachineSweep && machine) {
|
|
2047
|
+
throw new Error(`Fly Machine "${machine.id}" appeared during failed-create sweep; KEEP`);
|
|
2048
|
+
}
|
|
2049
|
+
if (!retire &&
|
|
2050
|
+
machine &&
|
|
2051
|
+
(activeStates.includes(machine.state ?? 'created') ||
|
|
2052
|
+
intent.volumes.some((volume) => volume.workspace))) {
|
|
2053
|
+
return false;
|
|
2054
|
+
}
|
|
2055
|
+
if (machine) {
|
|
2056
|
+
// Re-read immediately before the destructive decision, not from the lock-wait snapshot.
|
|
2057
|
+
intent = await fresh();
|
|
2058
|
+
live = await this.readAttemptAppState(intent.appName, signal);
|
|
2059
|
+
const current = ownMachine(live.machines, intent) ??
|
|
2060
|
+
(!forget || mayDeleteApp ? undefined : ((await readRecordedMachine(intent)) ?? undefined));
|
|
2061
|
+
if (failedMachineSweep && current) {
|
|
2062
|
+
throw new Error(`Fly Machine "${current.id}" appeared during failed-create sweep; KEEP`);
|
|
2063
|
+
}
|
|
2064
|
+
if (!retire &&
|
|
2065
|
+
current &&
|
|
2066
|
+
(activeStates.includes(current.state ?? 'created') ||
|
|
2067
|
+
intent.volumes.some((volume) => volume.workspace))) {
|
|
2068
|
+
return false;
|
|
2069
|
+
}
|
|
2070
|
+
if (automatic &&
|
|
2071
|
+
current &&
|
|
2072
|
+
(intent.machineCreatedByAttemptId !== intent.attemptId || intent.machineId !== current.id)) {
|
|
2073
|
+
throw new Error(`Fly Machine "${current.id}" for attempt "${intent.attemptId}" lost accepted-POST provenance before deletion; KEEP`);
|
|
2074
|
+
}
|
|
2075
|
+
if (current) {
|
|
2076
|
+
// Migration 1793275400000 recorded only the synthetic attempt and
|
|
2077
|
+
// sandbox ID. The owned Machine's mounts/routes rebuild the exact-ID
|
|
2078
|
+
// responsibilities before its metadata disappears on deletion.
|
|
2079
|
+
await adoptLegacyResources(current);
|
|
2080
|
+
intent = await fresh();
|
|
2081
|
+
// Observation is not ownership. Recheck every other durable reference
|
|
2082
|
+
// on this same app lock immediately before the irreversible request.
|
|
2083
|
+
await assertObservedMachineUnreferenced(intent, current.id);
|
|
2084
|
+
await this.deleteRetired(intent, generation, `/apps/${intent.appName}/machines/${current.id}?force=true`, signal);
|
|
2085
|
+
releasedMachineIds.add(current.id);
|
|
2086
|
+
}
|
|
2087
|
+
}
|
|
2088
|
+
const releasedVolumeIds = new Set();
|
|
2089
|
+
for (const recorded of intent.volumes) {
|
|
2090
|
+
intent = await fresh();
|
|
2091
|
+
let volumeIntent = intent.volumes.find((candidate) => candidate.name === recorded.name || candidate.id === recorded.id);
|
|
2092
|
+
if (!volumeIntent || (volumeIntent.resolved === true && !volumeIntent.id))
|
|
2093
|
+
continue;
|
|
2094
|
+
live = await this.readAttemptAppState(intent.appName, signal);
|
|
2095
|
+
if (live.appGone)
|
|
2096
|
+
continue;
|
|
2097
|
+
if (options.recordedResourcesOnly && !volumeIntent.id) {
|
|
2098
|
+
throw new Error(`Fly volume "${volumeIntent.name}" for attempt "${intent.attemptId}" has no exact durable ID; refusing discovery during explicit retirement`);
|
|
2099
|
+
}
|
|
2100
|
+
if (!volumeIntent.id) {
|
|
2101
|
+
const matches = live.volumes.filter((candidate) => candidate.name === volumeIntent?.name);
|
|
2102
|
+
if (matches.length > 1) {
|
|
2103
|
+
throw new Error(`Fly volume create "${volumeIntent.name}" has ambiguous matches; an explicit ID is required`);
|
|
2104
|
+
}
|
|
2105
|
+
const discovered = matches[0];
|
|
2106
|
+
if (!discovered) {
|
|
2107
|
+
throw new Error(`Fly volume create "${volumeIntent.name}" remains unresolved`);
|
|
2108
|
+
}
|
|
2109
|
+
// A unique name match is an observation only. Persist its exact ID for
|
|
2110
|
+
// deliberate retirement, never accepted-POST/automatic-delete authority.
|
|
2111
|
+
await journal.recordVolume(intent.attemptId, {
|
|
2112
|
+
...volumeIntent,
|
|
2113
|
+
id: discovered.id,
|
|
2114
|
+
resolved: true,
|
|
2115
|
+
createdByAttemptId: undefined,
|
|
2116
|
+
adoptedByAttemptId: intent.attemptId,
|
|
2117
|
+
});
|
|
2118
|
+
intent = await fresh();
|
|
2119
|
+
volumeIntent = intent.volumes.find((candidate) => candidate.id === discovered.id);
|
|
2120
|
+
}
|
|
2121
|
+
if (automatic &&
|
|
2122
|
+
volumeIntent &&
|
|
2123
|
+
(volumeIntent.createdByAttemptId !== intent.attemptId || volumeIntent.resolved !== true)) {
|
|
2124
|
+
throw new Error(`Fly volume "${volumeIntent.id ?? volumeIntent.name}" for attempt "${intent.attemptId}" has no matching accepted-POST provenance; KEEP`);
|
|
2125
|
+
}
|
|
2126
|
+
if (!volumeIntent?.id || (!retire && volumeIntent.workspace))
|
|
2127
|
+
continue;
|
|
2128
|
+
// ID, attachment, workspace flag and ownership are all re-read under the lock.
|
|
2129
|
+
intent = await fresh();
|
|
2130
|
+
volumeIntent = intent.volumes.find((candidate) => candidate.id === volumeIntent?.id);
|
|
2131
|
+
live = await this.readAttemptAppState(intent.appName, signal);
|
|
2132
|
+
const volume = live.volumes.find((candidate) => candidate.id === volumeIntent?.id);
|
|
2133
|
+
if (failedMachineSweep && volumeIntent?.id) {
|
|
2134
|
+
const referencing = live.machines.find((machine) => machine.config?.mounts?.some((mount) => mount.volume === volumeIntent?.id));
|
|
2135
|
+
if (referencing || volume?.attached_machine_id) {
|
|
2136
|
+
throw new Error(`Fly volume "${volumeIntent.id}" gained a Machine reference during sweep; KEEP`);
|
|
2137
|
+
}
|
|
2138
|
+
}
|
|
2139
|
+
if (automatic &&
|
|
2140
|
+
volumeIntent &&
|
|
2141
|
+
(volumeIntent.createdByAttemptId !== intent.attemptId || volumeIntent.resolved !== true)) {
|
|
2142
|
+
throw new Error(`Fly volume "${volumeIntent.id ?? volumeIntent.name}" for attempt "${intent.attemptId}" lost accepted-POST provenance before deletion; KEEP`);
|
|
2143
|
+
}
|
|
2144
|
+
if (!volumeIntent?.id || (!retire && volumeIntent.workspace) || volume?.attached_machine_id) {
|
|
2145
|
+
continue;
|
|
2146
|
+
}
|
|
2147
|
+
// A list omission is not deletion. The durable exact ID is sufficient
|
|
2148
|
+
// capability for an idempotent DELETE; success/404, never the list,
|
|
2149
|
+
// authorizes forgetting it. Fly still rejects a really attached volume.
|
|
2150
|
+
const volumeId = volumeIntent.id;
|
|
2151
|
+
await this.deleteRetired(intent, generation, `/apps/${intent.appName}/volumes/${volumeId}`, signal);
|
|
2152
|
+
releasedVolumeIds.add(volumeId);
|
|
2153
|
+
}
|
|
2154
|
+
intent = await fresh();
|
|
2155
|
+
live = await this.readAttemptAppState(intent.appName, signal);
|
|
2156
|
+
if (intent.addresses.length &&
|
|
2157
|
+
(live.machines.some((candidate) => candidate.id !== machine?.id) ||
|
|
2158
|
+
(await hasActiveSibling(intent)))) {
|
|
2159
|
+
return false;
|
|
2160
|
+
}
|
|
2161
|
+
for (const recorded of intent.addresses) {
|
|
2162
|
+
intent = await fresh();
|
|
2163
|
+
let address = intent.addresses.find((candidate) => candidate.targetApp === recorded.targetApp && candidate.network === recorded.network);
|
|
2164
|
+
if (!address || (address.resolved && !address.owned))
|
|
2165
|
+
continue;
|
|
2166
|
+
if (options.recordedResourcesOnly && (!address.owned || !address.ip)) {
|
|
2167
|
+
throw new Error(`Fly address for attempt "${intent.attemptId}" on "${address.targetApp}" has no exact owned IP; refusing discovery during explicit retirement`);
|
|
2168
|
+
}
|
|
2169
|
+
if (!address.resolved) {
|
|
2170
|
+
// Only an unresolved create needs discovery. A known owned IP is its
|
|
2171
|
+
// own DELETE capability, even when the target app no longer exists.
|
|
2172
|
+
const assignments = await this.listPrivateAssignments(address.targetApp, signal);
|
|
2173
|
+
if (!assignments) {
|
|
2174
|
+
throw new Error(`Could not resolve Fly addresses on "${address.targetApp}"`);
|
|
2175
|
+
}
|
|
2176
|
+
const assignment = assignments.find((candidate) => candidate.network === address?.network);
|
|
2177
|
+
if (!assignment?.ip) {
|
|
2178
|
+
throw new Error(`Fly address allocation for attempt "${intent.attemptId}" is unresolved`);
|
|
2179
|
+
}
|
|
2180
|
+
await journal.recordAddress(intent.attemptId, {
|
|
2181
|
+
...address,
|
|
2182
|
+
ip: assignment.ip,
|
|
2183
|
+
owned: true,
|
|
2184
|
+
resolved: true,
|
|
2185
|
+
});
|
|
2186
|
+
intent = await fresh();
|
|
2187
|
+
address = intent.addresses.find((candidate) => candidate.targetApp === recorded.targetApp && candidate.network === recorded.network);
|
|
2188
|
+
}
|
|
2189
|
+
if (!address?.owned)
|
|
2190
|
+
continue;
|
|
2191
|
+
if (!address.ip)
|
|
2192
|
+
throw new Error(`Owned Fly address for "${intent.attemptId}" has no IP`);
|
|
2193
|
+
// The durable exact IP, not list presence, is the release capability.
|
|
2194
|
+
// A retry after a lost successful DELETE gets 404 and completes below.
|
|
2195
|
+
intent = await fresh();
|
|
2196
|
+
address = intent.addresses.find((candidate) => candidate.targetApp === recorded.targetApp && candidate.network === recorded.network);
|
|
2197
|
+
if (!address?.owned)
|
|
2198
|
+
continue;
|
|
2199
|
+
if (!address.ip)
|
|
2200
|
+
throw new Error(`Owned Fly address for "${intent.attemptId}" has no IP`);
|
|
2201
|
+
await this.deleteRetired(intent, generation, `/apps/${address.targetApp}/ip_assignments/${encodeURIComponent(address.ip)}`, signal);
|
|
2202
|
+
await journal.recordAddress(intent.attemptId, {
|
|
2203
|
+
...address,
|
|
2204
|
+
owned: false,
|
|
2205
|
+
resolved: true,
|
|
2206
|
+
});
|
|
2207
|
+
}
|
|
2208
|
+
intent = await fresh();
|
|
2209
|
+
live = await this.readAttemptAppState(intent.appName, signal);
|
|
2210
|
+
const ownedVolumesRemain = intent.volumes.some((volume) => {
|
|
2211
|
+
if (volume.resolved === true && !volume.id)
|
|
2212
|
+
return false;
|
|
2213
|
+
if (live.appGone)
|
|
2214
|
+
return false;
|
|
2215
|
+
if (volume.id)
|
|
2216
|
+
return !releasedVolumeIds.has(volume.id);
|
|
2217
|
+
return live.volumes.some((candidate) => candidate.name === volume.name);
|
|
2218
|
+
});
|
|
2219
|
+
// Exact app 404 proves absence of every contained resource, including an
|
|
2220
|
+
// unresolved volume. External addresses still settle independently below.
|
|
2221
|
+
const unsettledVolume = !live.appGone && intent.volumes.some((volume) => volume.resolved !== true);
|
|
2222
|
+
if (failedMachineSweep && this.appPerProject() && live.machines.length) {
|
|
2223
|
+
throw new Error(`Fly app "${intent.appName}" gained a Machine during sweep; KEEP`);
|
|
2224
|
+
}
|
|
2225
|
+
if (failedMachineSweep && deleteApp && this.appPerProject() && live.volumes.length) {
|
|
2226
|
+
throw new Error(`Fly app "${intent.appName}" is not empty; failed-create sweep is KEEP`);
|
|
2227
|
+
}
|
|
2228
|
+
const listedMachine = ownMachine(live.machines, intent);
|
|
2229
|
+
if (unsettledVolume ||
|
|
2230
|
+
ownedVolumesRemain ||
|
|
2231
|
+
(listedMachine && !releasedMachineIds.has(listedMachine.id))) {
|
|
2232
|
+
return false;
|
|
2233
|
+
}
|
|
2234
|
+
if (intent.addresses.some((address) => !address.resolved || address.owned))
|
|
2235
|
+
return false;
|
|
2236
|
+
if (mayDeleteApp && !live.appGone && (await hasActiveSibling(intent))) {
|
|
2237
|
+
return false;
|
|
2238
|
+
}
|
|
2239
|
+
let appSettled = live.appGone;
|
|
2240
|
+
if (mayDeleteApp && !live.appGone) {
|
|
2241
|
+
// Deliberate retirement alone may delete a container. Addresses above are
|
|
2242
|
+
// settled first; provenance and every Machine are re-read under the lock.
|
|
2243
|
+
intent = await fresh();
|
|
2244
|
+
live = await this.readAttemptAppState(intent.appName, signal);
|
|
2245
|
+
if (failedMachineSweep && (live.machines.length || live.volumes.length)) {
|
|
2246
|
+
throw new Error(`Fly app "${intent.appName}" is not empty before app deletion; KEEP`);
|
|
2247
|
+
}
|
|
2248
|
+
if (automatic &&
|
|
2249
|
+
(intent.appCreatedByAttemptId !== intent.attemptId || intent.appCreateResolved !== true)) {
|
|
2250
|
+
throw new Error(`Fly app "${intent.appName}" for attempt "${intent.attemptId}" has no matching accepted-POST provenance; KEEP`);
|
|
2251
|
+
}
|
|
2252
|
+
if (live.machines.length || ((automatic || !retire) && live.volumes.length))
|
|
2253
|
+
return false;
|
|
2254
|
+
await this.deleteRetired(intent, generation, `/apps/${intent.appName}`, signal);
|
|
2255
|
+
appSettled = true;
|
|
2256
|
+
}
|
|
2257
|
+
if (automatic && appRequired && !appSettled && !this.hasUnresolvedMachineIdentity(intent)) {
|
|
2258
|
+
return false;
|
|
2259
|
+
}
|
|
2260
|
+
if (!forget)
|
|
2261
|
+
return false;
|
|
2262
|
+
intent = await fresh();
|
|
2263
|
+
await this.assertAuthoritativeForget(intent, signal, { machineIds: releasedMachineIds, volumeIds: releasedVolumeIds }, { required: appRequired, settled: appSettled });
|
|
2264
|
+
await journal.forget(intent.attemptId, generation);
|
|
2265
|
+
return true;
|
|
2266
|
+
}
|
|
2267
|
+
/** Lock and settle each eligible attempt independently; there is no lock-free mode. */
|
|
2268
|
+
async reconcileCreateAttempts(options = {}) {
|
|
2269
|
+
const journal = this.config.createAttemptJournal;
|
|
2270
|
+
if (!journal)
|
|
2271
|
+
return { deleted: 0, failed: 0 };
|
|
2272
|
+
const now = options.now ?? Date.now();
|
|
2273
|
+
const timeoutMs = options.createTimeoutMs ?? 15 * 60_000;
|
|
2274
|
+
const cutoff = new Date(now - timeoutMs);
|
|
2275
|
+
const intents = (await journal.reclaimable(cutoff)).filter((intent) => !options.attemptId || intent.attemptId === options.attemptId);
|
|
2276
|
+
const expiredCreatingIds = new Set(intents.filter((intent) => intent.state === 'creating').map((intent) => intent.attemptId));
|
|
2277
|
+
const summary = { deleted: 0, failed: 0 };
|
|
2278
|
+
for (const intent of intents) {
|
|
2279
|
+
try {
|
|
2280
|
+
if (!journal.withLock)
|
|
2281
|
+
throw new Error('Attempt settlement requires its lifecycle lock');
|
|
2282
|
+
const forgotten = await journal.withLock(intent, async (signal, generation) => {
|
|
2283
|
+
// Selection happened before the lock wait. Re-read state and expiry;
|
|
2284
|
+
// only failed/expired attempts may be retired by this automatic pass.
|
|
2285
|
+
const current = await journal.get(intent.attemptId);
|
|
2286
|
+
if (!current)
|
|
2287
|
+
return false;
|
|
2288
|
+
if (current.state === 'serving')
|
|
2289
|
+
return false;
|
|
2290
|
+
if (current.state === 'creating') {
|
|
2291
|
+
const expired = (await journal.reclaimable(cutoff)).some((candidate) => candidate.attemptId === intent.attemptId);
|
|
2292
|
+
if (!expired)
|
|
2293
|
+
return false;
|
|
2294
|
+
}
|
|
2295
|
+
if (signal.aborted)
|
|
2296
|
+
throw signal.reason;
|
|
2297
|
+
const failedMachineSweep = current.machineCreateFailed === true;
|
|
2298
|
+
if (failedMachineSweep)
|
|
2299
|
+
await this.assertFailedMachineSweepable(current, signal);
|
|
2300
|
+
if (current.state === 'failed' || current.state === 'creating') {
|
|
2301
|
+
await journal.retire(current.attemptId, generation);
|
|
2302
|
+
}
|
|
2303
|
+
if (signal.aborted)
|
|
2304
|
+
throw signal.reason;
|
|
2305
|
+
const settled = await this.settleCreateAttempt(current, signal, generation, {
|
|
2306
|
+
automatic: true,
|
|
2307
|
+
retire: failedMachineSweep,
|
|
2308
|
+
failedMachineSweep,
|
|
2309
|
+
expiredCreatingIds,
|
|
2310
|
+
});
|
|
2311
|
+
if (failedMachineSweep && !settled) {
|
|
2312
|
+
throw new Error(`Fly failed-create sweep for "${current.attemptId}" remains incomplete`);
|
|
2313
|
+
}
|
|
2314
|
+
return settled;
|
|
2315
|
+
});
|
|
2316
|
+
if (forgotten)
|
|
2317
|
+
summary.deleted++;
|
|
2318
|
+
}
|
|
2319
|
+
catch (error) {
|
|
2320
|
+
// A state transition back to creating/serving is a safe skip, not a cleanup failure.
|
|
2321
|
+
if (!/no longer settleable/.test(error instanceof Error ? error.message : '')) {
|
|
2322
|
+
summary.failed++;
|
|
2323
|
+
logger.warn('Could not reconcile a Fly create attempt; retaining its durable intent', {
|
|
2324
|
+
attemptId: intent.attemptId,
|
|
2325
|
+
app: intent.appName,
|
|
2326
|
+
error,
|
|
2327
|
+
});
|
|
2328
|
+
}
|
|
2329
|
+
}
|
|
2330
|
+
}
|
|
2331
|
+
return summary;
|
|
2332
|
+
}
|
|
2333
|
+
/** Re-read and abandon only while the attempt's current lifecycle lock is live. */
|
|
2334
|
+
async retireCreateAttempt(snapshot, outerSignal, options) {
|
|
2335
|
+
const journal = this.config.createAttemptJournal;
|
|
2336
|
+
if (!journal?.withLock)
|
|
2337
|
+
throw new Error('Attempt settlement requires its lifecycle lock');
|
|
2338
|
+
return journal.withLock(snapshot, async (lockSignal, generation) => {
|
|
2339
|
+
const signal = AbortSignal.any([outerSignal, lockSignal]);
|
|
2340
|
+
if (signal.aborted)
|
|
2341
|
+
throw signal.reason;
|
|
2342
|
+
const current = await journal.get(snapshot.attemptId);
|
|
2343
|
+
if (signal.aborted)
|
|
2344
|
+
throw signal.reason;
|
|
2345
|
+
if (!current)
|
|
2346
|
+
return false;
|
|
2347
|
+
if ((options.expectedAppName && current.appName !== options.expectedAppName) ||
|
|
2348
|
+
(options.expectedMachineId && this.recordedMachineId(current) !== options.expectedMachineId)) {
|
|
2349
|
+
throw new Error(`Fly create attempt "${snapshot.attemptId}" changed outside the caller-validated scope`);
|
|
2350
|
+
}
|
|
2351
|
+
if (current.state !== 'abandoned') {
|
|
2352
|
+
if (signal.aborted)
|
|
2353
|
+
throw signal.reason;
|
|
2354
|
+
await journal.retire(current.attemptId, generation);
|
|
2355
|
+
if (signal.aborted)
|
|
2356
|
+
throw signal.reason;
|
|
2357
|
+
}
|
|
2358
|
+
return this.settleCreateAttempt(current, signal, generation, {
|
|
2359
|
+
retire: true,
|
|
2360
|
+
...options,
|
|
2361
|
+
});
|
|
2362
|
+
});
|
|
2363
|
+
}
|
|
2364
|
+
/** Journal a pre-journal Machine before its first destructive request. */
|
|
2365
|
+
async journalLegacyMachine(app, machine, fallbackProjectId) {
|
|
2366
|
+
const journal = this.config.createAttemptJournal;
|
|
2367
|
+
if (!journal)
|
|
2368
|
+
throw new Error('Legacy Machine adoption requires its durable journal');
|
|
2369
|
+
const metadata = machine.config?.metadata ?? {};
|
|
2370
|
+
let projectId = metadata[`${this.metadataPrefix}.projectId`] ?? fallbackProjectId;
|
|
2371
|
+
if (!projectId && this.appPerProject()) {
|
|
2372
|
+
const prefix = `${this.appPrefix()}-`;
|
|
2373
|
+
const candidate = app.startsWith(prefix) ? app.slice(prefix.length) : '';
|
|
2374
|
+
if (candidate && appNameForProject(this.appPrefix(), candidate) === app) {
|
|
2375
|
+
projectId = candidate;
|
|
2376
|
+
}
|
|
2377
|
+
}
|
|
2378
|
+
if (!projectId) {
|
|
2379
|
+
throw new Error(`Refusing to destroy legacy Machine "${machine.id}" without its project identity`);
|
|
2380
|
+
}
|
|
2381
|
+
const sandboxId = `${app}:${machine.id}`;
|
|
2382
|
+
const attemptId = `legacy:${sandboxId}`;
|
|
2383
|
+
const existing = await journal.get(attemptId);
|
|
2384
|
+
if (existing) {
|
|
2385
|
+
await journal.markMachineAdopted(attemptId);
|
|
2386
|
+
if (existing.sandboxId) {
|
|
2387
|
+
const adopted = await journal.get(attemptId);
|
|
2388
|
+
if (!adopted)
|
|
2389
|
+
throw new Error(`Could not persist legacy ownership for "${sandboxId}"`);
|
|
2390
|
+
return adopted;
|
|
2391
|
+
}
|
|
2392
|
+
await journal.markState(attemptId, existing.state === 'creating' ? 'serving' : existing.state, sandboxId);
|
|
2393
|
+
const repaired = await journal.get(attemptId);
|
|
2394
|
+
if (!repaired?.sandboxId) {
|
|
2395
|
+
throw new Error(`Could not repair legacy ownership for "${sandboxId}"`);
|
|
2396
|
+
}
|
|
2397
|
+
return repaired;
|
|
2398
|
+
}
|
|
2399
|
+
await journal.begin({
|
|
2400
|
+
attemptId,
|
|
2401
|
+
projectId,
|
|
2402
|
+
ownerProjectId: metadata['molecule.production'] ?? projectId,
|
|
2403
|
+
appName: app,
|
|
2404
|
+
state: 'serving',
|
|
2405
|
+
sandboxId,
|
|
2406
|
+
machineAdopted: true,
|
|
2407
|
+
});
|
|
2408
|
+
const adopted = await journal.get(attemptId);
|
|
2409
|
+
if (!adopted)
|
|
2410
|
+
throw new Error(`Could not persist legacy ownership for "${sandboxId}"`);
|
|
2411
|
+
return adopted;
|
|
2412
|
+
}
|
|
2413
|
+
/** Journal an exclusively owned empty app rather than treating absence as DELETE authority. */
|
|
2414
|
+
async journalLegacyApp(app, projectId) {
|
|
2415
|
+
const journal = this.config.createAttemptJournal;
|
|
2416
|
+
if (!journal)
|
|
2417
|
+
throw new Error('Legacy app adoption requires its durable journal');
|
|
2418
|
+
const attemptId = `legacy:${app}:app`;
|
|
2419
|
+
const existing = await journal.get(attemptId);
|
|
2420
|
+
if (existing)
|
|
2421
|
+
return existing;
|
|
2422
|
+
await journal.begin({ attemptId, projectId, appName: app });
|
|
2423
|
+
await journal.markState(attemptId, 'serving');
|
|
2424
|
+
const adopted = await journal.get(attemptId);
|
|
2425
|
+
if (!adopted)
|
|
2426
|
+
throw new Error(`Could not persist legacy ownership for app "${app}"`);
|
|
2427
|
+
return adopted;
|
|
2428
|
+
}
|
|
2429
|
+
/**
|
|
2430
|
+
* Explicitly destroy an exclusively owned app and its workspace storage.
|
|
2431
|
+
* Recorded external addresses are released before any intent may be forgotten.
|
|
2432
|
+
*/
|
|
2433
|
+
async destroyApp(projectId, expectedAppName, signal) {
|
|
2434
|
+
if (!signal)
|
|
2435
|
+
throw new Error('Fly app destruction requires its lifecycle lock signal');
|
|
2436
|
+
if (!this.appPerProject()) {
|
|
2437
|
+
throw new Error('Refusing to delete a shared Fly app as though a caller exclusively owned it');
|
|
2438
|
+
}
|
|
2439
|
+
const app = appNameForProject(this.appPrefix(), projectId);
|
|
2440
|
+
if (expectedAppName !== undefined && expectedAppName !== app) {
|
|
2441
|
+
throw new Error(`Refusing to delete Fly app "${expectedAppName}": its durable identity resolves to "${app}"`);
|
|
2442
|
+
}
|
|
2443
|
+
const listed = await this.requestForDestruction(`/apps/${app}/machines`, { nullOn: [404] }, signal);
|
|
2444
|
+
const machines = Array.isArray(listed) ? listed : [];
|
|
2445
|
+
const routes = {};
|
|
2446
|
+
const attemptIds = new Set();
|
|
2447
|
+
const metadataHints = [];
|
|
2448
|
+
for (const machine of machines) {
|
|
2449
|
+
Object.assign(routes, decodePrivateRoutes(machine.config?.metadata?.[`${this.metadataPrefix}.${PRIVATE_ROUTES_METADATA_SUFFIX}`]));
|
|
2450
|
+
const attemptId = machine.config?.metadata?.[`${this.metadataPrefix}.createAttempt`];
|
|
2451
|
+
if (attemptId) {
|
|
2452
|
+
attemptIds.add(attemptId);
|
|
2453
|
+
metadataHints.push({ attemptId, machineId: machine.id });
|
|
2454
|
+
}
|
|
2455
|
+
}
|
|
2456
|
+
const journal = this.config.createAttemptJournal;
|
|
2457
|
+
if (journal) {
|
|
2458
|
+
const durable = new Map();
|
|
2459
|
+
const expectedMachineIds = new Map();
|
|
2460
|
+
const journaled = await journal.forApp(app);
|
|
2461
|
+
if (signal.aborted)
|
|
2462
|
+
throw signal.reason;
|
|
2463
|
+
for (const intent of journaled)
|
|
2464
|
+
durable.set(intent.attemptId, intent);
|
|
2465
|
+
for (const hint of metadataHints) {
|
|
2466
|
+
const intent = await journal.get(hint.attemptId);
|
|
2467
|
+
if (signal.aborted)
|
|
2468
|
+
throw signal.reason;
|
|
2469
|
+
// Metadata is only a hint. Both the requested app and this listed
|
|
2470
|
+
// Machine's exact durable identity must match before selection.
|
|
2471
|
+
if (intent?.appName === app && this.recordedMachineId(intent) === hint.machineId) {
|
|
2472
|
+
durable.set(intent.attemptId, intent);
|
|
2473
|
+
expectedMachineIds.set(intent.attemptId, hint.machineId);
|
|
2474
|
+
}
|
|
2475
|
+
else if (intent) {
|
|
2476
|
+
throw new Error(`Refusing copied createAttempt metadata for Machine "${hint.machineId}" in app "${app}"`);
|
|
2477
|
+
}
|
|
2478
|
+
}
|
|
2479
|
+
for (const machine of machines) {
|
|
2480
|
+
const createAttemptId = machine.config?.metadata?.[`${this.metadataPrefix}.createAttempt`];
|
|
2481
|
+
const known = [...durable.values()].some((intent) => intent.appName === app &&
|
|
2482
|
+
((intent.attemptId === createAttemptId &&
|
|
2483
|
+
this.recordedMachineId(intent) === machine.id) ||
|
|
2484
|
+
intent.sandboxId === `${app}:${machine.id}`));
|
|
2485
|
+
if (known)
|
|
2486
|
+
continue;
|
|
2487
|
+
const adopted = await this.journalLegacyMachine(app, machine, projectId);
|
|
2488
|
+
durable.set(adopted.attemptId, adopted);
|
|
2489
|
+
}
|
|
2490
|
+
if (!durable.size) {
|
|
2491
|
+
const adopted = await this.journalLegacyApp(app, projectId);
|
|
2492
|
+
durable.set(adopted.attemptId, adopted);
|
|
2493
|
+
}
|
|
2494
|
+
if (durable.size && !journal.withLock) {
|
|
2495
|
+
throw new Error(`Refusing to retire Fly app "${app}" without its lifecycle lock`);
|
|
2496
|
+
}
|
|
2497
|
+
if (durable.size && journal.withLock) {
|
|
2498
|
+
// Pass one settles every durable address and retires Machines without
|
|
2499
|
+
// deleting the app. No app deletion can bypass another attempt's failed
|
|
2500
|
+
// address release. Pass two deletes the app and forgets settled intents.
|
|
2501
|
+
for (const deleteApp of [false, true]) {
|
|
2502
|
+
for (const snapshot of durable.values()) {
|
|
2503
|
+
const current = await journal.get(snapshot.attemptId);
|
|
2504
|
+
if (signal.aborted)
|
|
2505
|
+
throw signal.reason;
|
|
2506
|
+
if (!current)
|
|
2507
|
+
continue;
|
|
2508
|
+
await this.retireCreateAttempt(current, signal, {
|
|
2509
|
+
deleteApp,
|
|
2510
|
+
forget: deleteApp,
|
|
2511
|
+
observedMachines: machines,
|
|
2512
|
+
expectedAppName: app,
|
|
2513
|
+
expectedMachineId: expectedMachineIds.get(current.attemptId),
|
|
2514
|
+
});
|
|
2515
|
+
}
|
|
2516
|
+
}
|
|
2517
|
+
const retained = (await Promise.all([...durable.keys()].map(async (attemptId) => journal.get(attemptId)))).some(Boolean);
|
|
2518
|
+
if (signal.aborted)
|
|
2519
|
+
throw signal.reason;
|
|
2520
|
+
if (retained)
|
|
2521
|
+
throw new Error(`Fly app "${app}" retirement has not fully settled`);
|
|
2522
|
+
return;
|
|
2523
|
+
}
|
|
2524
|
+
}
|
|
2525
|
+
// Legacy unjournaled apps retain explicit destruction. Never force an app
|
|
2526
|
+
// through a Machine that has no committed retired row: explicitly delete
|
|
2527
|
+
// each observed Machine, release its routes, then re-list before app DELETE.
|
|
2528
|
+
for (const machine of machines) {
|
|
2529
|
+
await this.requestForDestruction(`/apps/${app}/machines/${machine.id}?force=true`, { method: 'DELETE', nullOn: [404] }, signal);
|
|
2530
|
+
}
|
|
2531
|
+
for (const [targetApp, ip] of Object.entries(routes)) {
|
|
2532
|
+
if (!(await this.privateRouteMatchesNetwork(targetApp, ip, this.networkFor(app), signal))) {
|
|
2533
|
+
logger.warn('Kept a Fly private (Flycast) address with unvalidated network ownership', {
|
|
2534
|
+
targetApp,
|
|
2535
|
+
ip,
|
|
2536
|
+
app,
|
|
2537
|
+
});
|
|
2538
|
+
continue;
|
|
2539
|
+
}
|
|
2540
|
+
await this.requestForDestruction(`/apps/${targetApp}/ip_assignments/${encodeURIComponent(ip)}`, { method: 'DELETE', nullOn: [404] }, signal);
|
|
2541
|
+
}
|
|
2542
|
+
const remaining = await this.readAttemptAppState(app, signal);
|
|
2543
|
+
if (remaining.machines.length) {
|
|
2544
|
+
throw new Error(`Refusing to delete Fly app "${app}" while a Machine remains`);
|
|
2545
|
+
}
|
|
2546
|
+
if (!remaining.appGone) {
|
|
2547
|
+
await this.requestForDestruction(`/apps/${app}`, { method: 'DELETE', nullOn: [404] }, signal);
|
|
2548
|
+
}
|
|
2549
|
+
for (const attemptId of attemptIds) {
|
|
2550
|
+
if (signal.aborted)
|
|
2551
|
+
throw signal.reason;
|
|
2552
|
+
await journal?.forget(attemptId);
|
|
2553
|
+
}
|
|
2554
|
+
}
|
|
2555
|
+
/**
|
|
2556
|
+
* Deliberately retire one named shared-app attempt using only exact durable
|
|
2557
|
+
* resource IDs. Unlike legacy destroy(), this never infers ownership from a
|
|
2558
|
+
* Machine name, metadata, sandboxId, a volume name, or an address listing.
|
|
2559
|
+
*/
|
|
2560
|
+
async destroyAttempt(attemptId, signal) {
|
|
2561
|
+
if (!signal)
|
|
2562
|
+
throw new Error('Fly attempt destruction requires its lifecycle lock signal');
|
|
2563
|
+
if (this.appPerProject()) {
|
|
2564
|
+
throw new Error('Use explicit per-project app destruction for a per-project Fly attempt');
|
|
2565
|
+
}
|
|
2566
|
+
const journal = this.config.createAttemptJournal;
|
|
2567
|
+
if (!journal)
|
|
2568
|
+
throw new Error('Fly attempt destruction requires its durable journal');
|
|
2569
|
+
const snapshot = await journal.get(attemptId);
|
|
2570
|
+
if (signal.aborted)
|
|
2571
|
+
throw signal.reason;
|
|
2572
|
+
if (!snapshot)
|
|
2573
|
+
return;
|
|
2574
|
+
const forgotten = await this.retireCreateAttempt(snapshot, signal, {
|
|
2575
|
+
deleteApp: false,
|
|
2576
|
+
forget: true,
|
|
2577
|
+
recordedResourcesOnly: true,
|
|
2578
|
+
});
|
|
2579
|
+
if (!forgotten) {
|
|
2580
|
+
throw new Error(`Could not complete exact-resource cleanup for attempt "${attemptId}"; its intent was retained`);
|
|
2581
|
+
}
|
|
2582
|
+
}
|
|
1490
2583
|
/**
|
|
1491
2584
|
* Destroys a sandbox.
|
|
1492
2585
|
*
|
|
@@ -1504,15 +2597,19 @@ class FlyioSandboxProvider {
|
|
|
1504
2597
|
* other than a 404 (which counts as already gone), naming the failed step and
|
|
1505
2598
|
* HTTP status, so the caller can retry instead of forgetting a billed app.
|
|
1506
2599
|
*/
|
|
1507
|
-
async destroy(id) {
|
|
2600
|
+
async destroy(id, signal) {
|
|
2601
|
+
if (!signal)
|
|
2602
|
+
throw new Error('Fly destruction requires its lifecycle lock signal');
|
|
1508
2603
|
const { app, machineId } = parseSandboxId(id);
|
|
1509
2604
|
let volumeIds = [];
|
|
1510
2605
|
let privateRoutes = {};
|
|
2606
|
+
let createAttemptId;
|
|
2607
|
+
let observedMachine = null;
|
|
1511
2608
|
try {
|
|
1512
|
-
const machine = await this.
|
|
1513
|
-
|
|
1514
|
-
});
|
|
2609
|
+
const machine = await this.requestForDestruction(`/apps/${app}/machines/${machineId}`, { nullOn: [404] }, signal);
|
|
2610
|
+
observedMachine = machine;
|
|
1515
2611
|
privateRoutes = decodePrivateRoutes(machine?.config?.metadata?.[`${this.metadataPrefix}.${PRIVATE_ROUTES_METADATA_SUFFIX}`]);
|
|
2612
|
+
createAttemptId = machine?.config?.metadata?.[`${this.metadataPrefix}.createAttempt`];
|
|
1516
2613
|
if (!this.appPerProject()) {
|
|
1517
2614
|
volumeIds = (machine?.config?.mounts ?? [])
|
|
1518
2615
|
.map((mount) => mount.volume)
|
|
@@ -1520,19 +2617,60 @@ class FlyioSandboxProvider {
|
|
|
1520
2617
|
}
|
|
1521
2618
|
}
|
|
1522
2619
|
catch (error) {
|
|
2620
|
+
if (signal.aborted)
|
|
2621
|
+
throw signal.reason;
|
|
1523
2622
|
// Best-effort metadata gathering; the Machine may already be gone.
|
|
1524
2623
|
// Destruction proceeds either way — but say so, because an unread
|
|
1525
2624
|
// metadata block means the private addresses below are NOT released.
|
|
1526
2625
|
logger.warn('Could not read the Fly Machine before destroy — its volumes and any private (Flycast) ' +
|
|
1527
2626
|
'addresses it recorded will not be cleaned up', { id, error });
|
|
1528
2627
|
}
|
|
2628
|
+
const journal = this.config.createAttemptJournal;
|
|
2629
|
+
if (journal) {
|
|
2630
|
+
const durable = await journal.forApp(app);
|
|
2631
|
+
if (signal.aborted)
|
|
2632
|
+
throw signal.reason;
|
|
2633
|
+
// Metadata is only a lookup hint. It selects an attempt when its durable
|
|
2634
|
+
// exact Machine identity also names the caller's ID; copied metadata must
|
|
2635
|
+
// never redirect retirement to a different Machine's attempt.
|
|
2636
|
+
let intent = durable.find((candidate) => candidate.attemptId === createAttemptId &&
|
|
2637
|
+
this.recordedMachineId(candidate) === machineId) ?? durable.find((candidate) => candidate.sandboxId === id);
|
|
2638
|
+
if (!intent && observedMachine) {
|
|
2639
|
+
intent = await this.journalLegacyMachine(app, observedMachine);
|
|
2640
|
+
}
|
|
2641
|
+
if (intent) {
|
|
2642
|
+
const current = await journal.get(intent.attemptId);
|
|
2643
|
+
if (signal.aborted)
|
|
2644
|
+
throw signal.reason;
|
|
2645
|
+
if (!current)
|
|
2646
|
+
throw new Error(`Fly create attempt "${intent.attemptId}" disappeared`);
|
|
2647
|
+
const durableMachineId = this.recordedMachineId(current);
|
|
2648
|
+
if (durableMachineId !== machineId) {
|
|
2649
|
+
throw new Error(`Refusing to retire attempt "${current.attemptId}": caller named Machine "${machineId}" but durable identity is "${durableMachineId ?? 'unresolved'}"`);
|
|
2650
|
+
}
|
|
2651
|
+
const forgotten = await this.retireCreateAttempt(current, signal, {
|
|
2652
|
+
deleteApp: this.appPerProject(),
|
|
2653
|
+
forget: true,
|
|
2654
|
+
observedMachines: observedMachine ? [observedMachine] : [],
|
|
2655
|
+
expectedAppName: app,
|
|
2656
|
+
expectedMachineId: machineId,
|
|
2657
|
+
});
|
|
2658
|
+
if (!forgotten) {
|
|
2659
|
+
throw new Error(`Could not complete every cleanup step for "${id}"; its intent was retained`);
|
|
2660
|
+
}
|
|
2661
|
+
return;
|
|
2662
|
+
}
|
|
2663
|
+
if (durable.length) {
|
|
2664
|
+
throw new Error(`Refusing to destroy "${id}" without its durable ownership record; ${durable.length} other attempt(s) remain`);
|
|
2665
|
+
}
|
|
2666
|
+
throw new Error(`Refusing to destroy "${id}" before its legacy ownership can be journaled`);
|
|
2667
|
+
}
|
|
1529
2668
|
try {
|
|
1530
|
-
await this.
|
|
1531
|
-
method: 'DELETE',
|
|
1532
|
-
nullOn: [404],
|
|
1533
|
-
});
|
|
2669
|
+
await this.requestForDestruction(`/apps/${app}/machines/${machineId}?force=true`, { method: 'DELETE', nullOn: [404] }, signal);
|
|
1534
2670
|
}
|
|
1535
2671
|
catch (error) {
|
|
2672
|
+
if (signal.aborted)
|
|
2673
|
+
throw signal.reason;
|
|
1536
2674
|
// The app delete is not attempted: removing the app under a Machine that
|
|
1537
2675
|
// is still alive would hide it. The private addresses are left for the
|
|
1538
2676
|
// retry, which re-reads them from the Machine's metadata.
|
|
@@ -1542,7 +2680,13 @@ class FlyioSandboxProvider {
|
|
|
1542
2680
|
if (this.appPerProject()) {
|
|
1543
2681
|
let appError;
|
|
1544
2682
|
try {
|
|
1545
|
-
await this.
|
|
2683
|
+
const remaining = await this.readAttemptAppState(app, signal);
|
|
2684
|
+
if (remaining.machines.length) {
|
|
2685
|
+
throw new Error(`Refusing to delete Fly app "${app}" while a Machine remains`);
|
|
2686
|
+
}
|
|
2687
|
+
if (!remaining.appGone) {
|
|
2688
|
+
await this.requestForDestruction(`/apps/${app}`, { method: 'DELETE', nullOn: [404] }, signal);
|
|
2689
|
+
}
|
|
1546
2690
|
}
|
|
1547
2691
|
catch (error) {
|
|
1548
2692
|
appError = error;
|
|
@@ -1553,24 +2697,41 @@ class FlyioSandboxProvider {
|
|
|
1553
2697
|
}
|
|
1554
2698
|
// Released even when the app delete failed: the Machine is gone, so a
|
|
1555
2699
|
// retry could no longer read these addresses from its metadata.
|
|
1556
|
-
await this.releasePrivateRoutes(privateRoutes);
|
|
2700
|
+
const routesReleased = await this.releasePrivateRoutes(privateRoutes, signal, this.networkFor(app));
|
|
1557
2701
|
if (appError !== undefined) {
|
|
1558
2702
|
throw new Error(describeDestroyFailure('app delete', id, appError), { cause: appError });
|
|
1559
2703
|
}
|
|
2704
|
+
if (!routesReleased && createAttemptId && this.config.createAttemptJournal) {
|
|
2705
|
+
throw new Error(`Could not release every external Flycast address for "${id}"`);
|
|
2706
|
+
}
|
|
2707
|
+
if (signal.aborted)
|
|
2708
|
+
throw signal.reason;
|
|
2709
|
+
if (createAttemptId)
|
|
2710
|
+
await this.config.createAttemptJournal?.forget(createAttemptId);
|
|
1560
2711
|
return;
|
|
1561
2712
|
}
|
|
2713
|
+
let cleanupFailed = false;
|
|
1562
2714
|
for (const volumeId of volumeIds) {
|
|
1563
2715
|
try {
|
|
1564
|
-
await this.
|
|
1565
|
-
method: 'DELETE',
|
|
1566
|
-
nullOn: [404],
|
|
1567
|
-
});
|
|
2716
|
+
await this.requestForDestruction(`/apps/${app}/volumes/${volumeId}`, { method: 'DELETE', nullOn: [404] }, signal);
|
|
1568
2717
|
}
|
|
1569
2718
|
catch (error) {
|
|
2719
|
+
if (signal.aborted)
|
|
2720
|
+
throw signal.reason;
|
|
2721
|
+
cleanupFailed = true;
|
|
1570
2722
|
logger.warn('Failed to remove Fly volume', { app, volumeId, error });
|
|
1571
2723
|
}
|
|
1572
2724
|
}
|
|
1573
|
-
|
|
2725
|
+
cleanupFailed =
|
|
2726
|
+
!(await this.releasePrivateRoutes(privateRoutes, signal, this.networkFor(app))) ||
|
|
2727
|
+
cleanupFailed;
|
|
2728
|
+
if (cleanupFailed && createAttemptId && this.config.createAttemptJournal) {
|
|
2729
|
+
throw new Error(`Could not complete every cleanup step for "${id}"; its intent was retained`);
|
|
2730
|
+
}
|
|
2731
|
+
if (signal.aborted)
|
|
2732
|
+
throw signal.reason;
|
|
2733
|
+
if (createAttemptId)
|
|
2734
|
+
await this.config.createAttemptJournal?.forget(createAttemptId);
|
|
1574
2735
|
}
|
|
1575
2736
|
// ---------------------------------------------------------------------------
|
|
1576
2737
|
// Templates
|
|
@@ -1649,19 +2810,15 @@ class FlyioSandboxProvider {
|
|
|
1649
2810
|
return templates.removeTemplate(this.templateContext(), templateId);
|
|
1650
2811
|
}
|
|
1651
2812
|
/**
|
|
1652
|
-
* Restores a template into a freshly-created Machine.
|
|
1653
|
-
*
|
|
1654
|
-
* A failure here DESTROYS the Machine. The alternative is to hand back a
|
|
1655
|
-
* sandbox that started cleanly and whose filesystem is not the one that was
|
|
1656
|
-
* asked for — the same silent-wrong-contents failure that makes a missing
|
|
1657
|
-
* template a hard error rather than a fallback to the base image.
|
|
2813
|
+
* Restores a template into a freshly-created Machine. A failure is journaled
|
|
2814
|
+
* by create and settled by the reconciler, never by an inline delete.
|
|
1658
2815
|
* @param app - The Fly app name.
|
|
1659
2816
|
* @param machineId - The Fly Machine id.
|
|
1660
2817
|
* @param templateId - The caller's template identifier.
|
|
1661
2818
|
* @param manifest - The CONTROL-PLANE manifest, whose `capturePaths` bound what is extracted.
|
|
1662
|
-
* @throws {Error} When the restore fails
|
|
2819
|
+
* @throws {Error} When the restore fails.
|
|
1663
2820
|
*/
|
|
1664
|
-
async restoreTemplate(app, machineId, templateId, manifest) {
|
|
2821
|
+
async restoreTemplate(app, machineId, templateId, manifest, owner) {
|
|
1665
2822
|
const ctx = this.templateContext();
|
|
1666
2823
|
const store = templates.requireStore(ctx);
|
|
1667
2824
|
const leaseId = `${machineId}-${Date.now().toString(36)}`.replace(/[^A-Za-z0-9_.-]/g, '_');
|
|
@@ -1669,10 +2826,10 @@ class FlyioSandboxProvider {
|
|
|
1669
2826
|
try {
|
|
1670
2827
|
// The restore runs through exec, so the Machine has to be up. `create`
|
|
1671
2828
|
// launches it but returns as soon as Fly accepts the request.
|
|
1672
|
-
await this.waitForStarted(app, machineId);
|
|
2829
|
+
await this.waitForStarted(app, machineId, owner);
|
|
1673
2830
|
lease = await templates.acquireRestoreLease(ctx, templateId, leaseId);
|
|
1674
2831
|
const url = await store.presignGet(templates.archiveKey(store, templateId), ctx.presignExpirySeconds);
|
|
1675
|
-
const result = await
|
|
2832
|
+
const result = await execCommand(this.rawExec(app, machineId, owner), templates.buildRestoreCommand(manifest.capturePaths, url), { cwd: '/', timeout: ctx.transferTimeoutMs }, '/');
|
|
1676
2833
|
if (result.exitCode !== 0) {
|
|
1677
2834
|
throw new Error(t('codeSandbox.flyio.error.templateRestoreFailed', { templateId, exitCode: String(result.exitCode) }, {
|
|
1678
2835
|
defaultValue: `Restoring template "${templateId}" into Fly Machine ${machineId} failed ` +
|
|
@@ -1680,19 +2837,6 @@ class FlyioSandboxProvider {
|
|
|
1680
2837
|
}));
|
|
1681
2838
|
}
|
|
1682
2839
|
}
|
|
1683
|
-
catch (error) {
|
|
1684
|
-
try {
|
|
1685
|
-
await this.client.request(`/apps/${app}/machines/${machineId}?force=true`, {
|
|
1686
|
-
method: 'DELETE',
|
|
1687
|
-
nullOn: [404],
|
|
1688
|
-
});
|
|
1689
|
-
}
|
|
1690
|
-
catch (cleanupError) {
|
|
1691
|
-
logger.warn('Failed to destroy the Fly Machine whose template restore failed — it will keep ' +
|
|
1692
|
-
'billing until it is reaped', { app, machineId, error: cleanupError });
|
|
1693
|
-
}
|
|
1694
|
-
throw error;
|
|
1695
|
-
}
|
|
1696
2840
|
finally {
|
|
1697
2841
|
if (lease)
|
|
1698
2842
|
await templates.releaseRestoreLease(ctx, lease);
|
|
@@ -1758,15 +2902,32 @@ class FlyioSandboxProvider {
|
|
|
1758
2902
|
: DEFAULT_PROBE_TIMEOUT_MS;
|
|
1759
2903
|
let app = null;
|
|
1760
2904
|
let machineId = null;
|
|
2905
|
+
let ownsProbeApp = false;
|
|
1761
2906
|
let policyPorts;
|
|
1762
2907
|
try {
|
|
1763
2908
|
policyPorts = this.egressAllowedPorts();
|
|
1764
|
-
|
|
1765
|
-
//
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
2909
|
+
// A random name collision is an adoption, never DELETE authority. Retry
|
|
2910
|
+
// fresh names in per-project mode; shared mode keeps its designated app.
|
|
2911
|
+
for (let attempt = 0; attempt < 5; attempt++) {
|
|
2912
|
+
app = this.resolveApp(this.probeProjectId());
|
|
2913
|
+
const ownership = { created: false };
|
|
2914
|
+
// The SAME policy a real sandbox gets (private-service ports included),
|
|
2915
|
+
// but NO Flycast allocation. Accepted app-create ownership is captured
|
|
2916
|
+
// before any later policy/Machine step can fail.
|
|
2917
|
+
await this.ensureApp(app, {
|
|
2918
|
+
privateRoutes: false,
|
|
2919
|
+
requireCreated: this.appPerProject(),
|
|
2920
|
+
ownership,
|
|
2921
|
+
});
|
|
2922
|
+
ownsProbeApp = ownership.created;
|
|
2923
|
+
if (!this.appPerProject() || ownsProbeApp)
|
|
2924
|
+
break;
|
|
2925
|
+
}
|
|
2926
|
+
if (this.appPerProject() && !ownsProbeApp) {
|
|
2927
|
+
throw new Error('Could not allocate a fresh Fly egress probe app name');
|
|
2928
|
+
}
|
|
2929
|
+
if (!app)
|
|
2930
|
+
throw new Error('Could not resolve a Fly egress probe app');
|
|
1770
2931
|
machineId = await this.createProbeMachine(app);
|
|
1771
2932
|
await this.waitForStarted(app, machineId);
|
|
1772
2933
|
const seconds = Math.min(MAX_WAIT_SECONDS, Math.max(1, Math.ceil(timeoutMs / 1000) + PROBE_EXEC_HEADROOM_SECONDS));
|
|
@@ -1794,7 +2955,7 @@ class FlyioSandboxProvider {
|
|
|
1794
2955
|
};
|
|
1795
2956
|
}
|
|
1796
2957
|
finally {
|
|
1797
|
-
await this.destroyProbe(app, machineId);
|
|
2958
|
+
await this.destroyProbe(app, machineId, ownsProbeApp);
|
|
1798
2959
|
}
|
|
1799
2960
|
}
|
|
1800
2961
|
/**
|
|
@@ -1855,15 +3016,17 @@ class FlyioSandboxProvider {
|
|
|
1855
3016
|
return machine.id;
|
|
1856
3017
|
}
|
|
1857
3018
|
/**
|
|
1858
|
-
* Removes the probe Machine and, in per-project mode,
|
|
3019
|
+
* Removes the accepted probe Machine and, in per-project mode, only an app
|
|
3020
|
+
* whose create POST this probe itself had accepted. Adopted/unknown apps KEEP.
|
|
1859
3021
|
*
|
|
1860
3022
|
* Best-effort by design: a cleanup failure must not turn a completed
|
|
1861
3023
|
* observation into an error, and `auto_destroy` plus the bounded init sleep
|
|
1862
3024
|
* already reap the Machine on their own.
|
|
1863
3025
|
* @param app - The probe app, or `null` when it was never resolved.
|
|
1864
|
-
* @param machineId - The probe Machine id, or `null` when
|
|
3026
|
+
* @param machineId - The accepted probe Machine id, or `null` when none was created.
|
|
3027
|
+
* @param appCreated - Accepted app-create provenance for this exact probe.
|
|
1865
3028
|
*/
|
|
1866
|
-
async destroyProbe(app, machineId) {
|
|
3029
|
+
async destroyProbe(app, machineId, appCreated) {
|
|
1867
3030
|
if (!app)
|
|
1868
3031
|
return;
|
|
1869
3032
|
if (machineId) {
|
|
@@ -1877,8 +3040,8 @@ class FlyioSandboxProvider {
|
|
|
1877
3040
|
logger.warn('Failed to remove the Fly egress probe Machine', { app, machineId, error });
|
|
1878
3041
|
}
|
|
1879
3042
|
}
|
|
1880
|
-
// Never in shared-app mode
|
|
1881
|
-
if (!this.appPerProject())
|
|
3043
|
+
// Never in shared-app mode, nor for a colliding/adopted/unknown app.
|
|
3044
|
+
if (!this.appPerProject() || !appCreated)
|
|
1882
3045
|
return;
|
|
1883
3046
|
try {
|
|
1884
3047
|
await this.client.request(`/apps/${app}`, { method: 'DELETE', nullOn: [404] });
|
|
@@ -1940,12 +3103,13 @@ class FlyioSandboxProvider {
|
|
|
1940
3103
|
* @param machineId - The Fly Machine id.
|
|
1941
3104
|
* @returns A transport callback for {@link execCommand}.
|
|
1942
3105
|
*/
|
|
1943
|
-
rawExec(app, machineId) {
|
|
3106
|
+
rawExec(app, machineId, owner) {
|
|
1944
3107
|
return async (command, timeoutSeconds) => {
|
|
1945
3108
|
const deadline = Date.now() + EXEC_PRECONDITION_BUDGET_MS;
|
|
1946
3109
|
for (let attempt = 1;; attempt++) {
|
|
1947
3110
|
try {
|
|
1948
|
-
const
|
|
3111
|
+
const path = `/apps/${app}/machines/${machineId}/exec`;
|
|
3112
|
+
const request = {
|
|
1949
3113
|
method: 'POST',
|
|
1950
3114
|
body: { command, timeout: timeoutSeconds },
|
|
1951
3115
|
// The API's own timeout bounds the command; allow the HTTP request a
|
|
@@ -1955,7 +3119,10 @@ class FlyioSandboxProvider {
|
|
|
1955
3119
|
// exec burst importFiles issues) is Fly API inconsistency under load,
|
|
1956
3120
|
// not a real absence — retry it alongside the default 429/5xx.
|
|
1957
3121
|
retryStatuses: [404],
|
|
1958
|
-
}
|
|
3122
|
+
};
|
|
3123
|
+
const response = owner
|
|
3124
|
+
? await this.requestForAttempt(owner, path, request)
|
|
3125
|
+
: await this.client.request(path, request);
|
|
1959
3126
|
return response ?? {};
|
|
1960
3127
|
}
|
|
1961
3128
|
catch (error) {
|
|
@@ -2101,7 +3268,7 @@ class FlyioSandboxProvider {
|
|
|
2101
3268
|
* @param machineId - The Fly Machine id.
|
|
2102
3269
|
* @throws {Error} When the Machine has not started within the budget.
|
|
2103
3270
|
*/
|
|
2104
|
-
async waitForStarted(app, machineId) {
|
|
3271
|
+
async waitForStarted(app, machineId, owner) {
|
|
2105
3272
|
const totalSeconds = Math.max(1, this.config.startTimeoutSeconds ?? DEFAULT_START_TIMEOUT_SECONDS);
|
|
2106
3273
|
const deadline = Date.now() + totalSeconds * 1000;
|
|
2107
3274
|
let lastState;
|
|
@@ -2111,24 +3278,45 @@ class FlyioSandboxProvider {
|
|
|
2111
3278
|
// enough: a host that does not have the image cached (every first boot
|
|
2112
3279
|
// after an image push) pulls it first, which alone can exceed 60 s.
|
|
2113
3280
|
for (;;) {
|
|
3281
|
+
if (owner) {
|
|
3282
|
+
await this.config.createAttemptJournal?.assertGeneration(owner.attemptId, owner.generation);
|
|
3283
|
+
if (owner.signal.aborted)
|
|
3284
|
+
throw owner.signal.reason;
|
|
3285
|
+
}
|
|
2114
3286
|
const remaining = Math.ceil((deadline - Date.now()) / 1000);
|
|
2115
3287
|
const seconds = Math.max(1, Math.min(remaining, MAX_WAIT_SECONDS));
|
|
2116
3288
|
try {
|
|
2117
3289
|
await this.client.request(`/apps/${app}/machines/${machineId}/wait?state=started&timeout=${seconds}`,
|
|
2118
3290
|
// One attempt: a retry would re-block for the whole round, and the
|
|
2119
3291
|
// state re-read below decides whether another round is warranted.
|
|
2120
|
-
{ attempts: 1, timeoutMs: (seconds + 15) * 1000 });
|
|
3292
|
+
{ attempts: 1, timeoutMs: (seconds + 15) * 1000, signal: owner?.signal });
|
|
3293
|
+
if (owner) {
|
|
3294
|
+
await this.config.createAttemptJournal?.assertGeneration(owner.attemptId, owner.generation);
|
|
3295
|
+
if (owner.signal.aborted)
|
|
3296
|
+
throw owner.signal.reason;
|
|
3297
|
+
}
|
|
2121
3298
|
return;
|
|
2122
3299
|
}
|
|
2123
3300
|
catch (error) {
|
|
3301
|
+
if (owner?.signal.aborted)
|
|
3302
|
+
throw owner.signal.reason;
|
|
2124
3303
|
lastError = error;
|
|
2125
3304
|
// The wait endpoint returning an error is not itself proof the Machine
|
|
2126
3305
|
// failed — re-read the authoritative state before deciding.
|
|
2127
3306
|
const machine = await this.client
|
|
2128
|
-
.request(`/apps/${app}/machines/${machineId}`, {
|
|
3307
|
+
.request(`/apps/${app}/machines/${machineId}`, {
|
|
3308
|
+
nullOn: [404],
|
|
3309
|
+
signal: owner?.signal,
|
|
3310
|
+
})
|
|
2129
3311
|
.catch(() => null);
|
|
2130
|
-
if (machine?.state === 'started')
|
|
3312
|
+
if (machine?.state === 'started') {
|
|
3313
|
+
if (owner) {
|
|
3314
|
+
await this.config.createAttemptJournal?.assertGeneration(owner.attemptId, owner.generation);
|
|
3315
|
+
if (owner.signal.aborted)
|
|
3316
|
+
throw owner.signal.reason;
|
|
3317
|
+
}
|
|
2131
3318
|
return;
|
|
3319
|
+
}
|
|
2132
3320
|
lastState = machine?.state;
|
|
2133
3321
|
// Another round is useful only while the Machine is still on its way
|
|
2134
3322
|
// up (`created` = provisioning/image pull, `starting` = boot). Any
|