@molecule/api-code-sandbox-flyio 1.3.6 → 1.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +359 -163
- package/dist/api.d.ts +17 -2
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +24 -3
- package/dist/ids.d.ts +9 -10
- package/dist/ids.d.ts.map +1 -1
- package/dist/ids.js +32 -18
- package/dist/index.d.ts +33 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +33 -4
- package/dist/lifecycle.d.ts +142 -0
- package/dist/lifecycle.d.ts.map +1 -0
- package/dist/lifecycle.js +14 -0
- package/dist/provider.d.ts +2 -1
- package/dist/provider.d.ts.map +1 -1
- package/dist/provider.js +1484 -306
- package/dist/types.d.ts +33 -7
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +0 -10
- package/package.json +2 -2
package/dist/provider.js
CHANGED
|
@@ -6,16 +6,16 @@
|
|
|
6
6
|
*
|
|
7
7
|
* @module
|
|
8
8
|
*/
|
|
9
|
-
import { createHash } from 'node:crypto';
|
|
9
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
10
10
|
import { promisify } from 'node:util';
|
|
11
11
|
import { gzip } from 'node:zlib';
|
|
12
12
|
import { getLogger } from '@molecule/api-bond';
|
|
13
13
|
import { t } from '@molecule/api-i18n';
|
|
14
|
-
import { FlyApiClient, FlyApiError, normalizeApiUrl } from './api.js';
|
|
14
|
+
import { FlyApiClient, FlyApiError, normalizeApiUrl, retryDelayMs, } from './api.js';
|
|
15
15
|
import { buildEgressProbeCommand, DEFAULT_EGRESS_POLICY_NAME, DEFAULT_EGRESS_PROBE_TARGETS, describePublicReach, extractPolicyId, extractPolicyPorts, parseEgressAllowedPorts, parseEgressProbeTargets, verdictForProbeExit, } from './egress.js';
|
|
16
16
|
import { execCommand, toExecResult } from './exec.js';
|
|
17
17
|
import { assertPrivateRoutesForEnv, decodePrivateRoutes, DEFAULT_PRIVATE_IP_TYPE, encodePrivateRoutes, flycastHost, mergeEgressPorts, parsePrivateServices, PRIVATE_ROUTES_METADATA_SUFFIX, } from './flycast.js';
|
|
18
|
-
import { appNameForProject, mapMachineState, parseSandboxId, toSandboxId } from './ids.js';
|
|
18
|
+
import { appNameForProject, appNameNamespace, mapMachineState, parseSandboxId, toSandboxId, } from './ids.js';
|
|
19
19
|
import { createTemplateStore, MAX_PRESIGN_EXPIRY_SECONDS } from './storage.js';
|
|
20
20
|
import * as templates from './templates.js';
|
|
21
21
|
import { shellQuote } from './utilities.js';
|
|
@@ -102,6 +102,8 @@ const DEFAULT_PREVIEW_PORT = 5173;
|
|
|
102
102
|
const WORKSPACE_PATH = '/workspace';
|
|
103
103
|
/** Default volume size in GB when `SandboxConfig.resources.diskMB` is unset. */
|
|
104
104
|
const DEFAULT_VOLUME_GB = 10;
|
|
105
|
+
/** Exact-identity reads allowed after an ambiguous create before its outcome is declared unknown. */
|
|
106
|
+
const CREATE_RECONCILE_ATTEMPTS = 3;
|
|
105
107
|
/** Page size for the org-wide Machine listing. */
|
|
106
108
|
const LIST_PAGE_SIZE = 200;
|
|
107
109
|
/** Hard cap on pages fetched while listing, so a bad cursor cannot loop forever. */
|
|
@@ -204,13 +206,13 @@ const MACHINE_CREATE_ATTEMPTS = 6;
|
|
|
204
206
|
* means no Machine was created — unlike a 5xx/timeout, where one might have
|
|
205
207
|
* been. 412 is safe for the same reason: "volume already claimed by machine
|
|
206
208
|
* <id>" refuses the create. It is what the create answers in the seconds after
|
|
207
|
-
*
|
|
208
|
-
*
|
|
209
|
-
*
|
|
209
|
+
* an explicit destroy released the Machine that held the volume — Fly releases
|
|
210
|
+
* the claim asynchronously, and an immediate create lost a production deploy
|
|
211
|
+
* to it (2026-10-14; the identical retry a
|
|
210
212
|
* minute later succeeded). Only these two are added on top of the client's
|
|
211
213
|
* default retryable set; neither retry can double-create.
|
|
212
214
|
*/
|
|
213
|
-
const MACHINE_CREATE_RETRY_STATUSES = [404, 412];
|
|
215
|
+
const MACHINE_CREATE_RETRY_STATUSES = [400, 404, 412];
|
|
214
216
|
/**
|
|
215
217
|
* What Fly answers when it reached the exec endpoint but the connection to the
|
|
216
218
|
* Machine's exec agent died before the command ran: `412 {"error":
|
|
@@ -369,8 +371,11 @@ class FlyioSandboxProvider {
|
|
|
369
371
|
* Resolves the Fly region for Machines and volumes.
|
|
370
372
|
* @returns The region code.
|
|
371
373
|
*/
|
|
372
|
-
region() {
|
|
373
|
-
|
|
374
|
+
region(createRegion) {
|
|
375
|
+
// FLY_REGION is the API Machine's own location on Fly, not a tenant
|
|
376
|
+
// placement decision. Only provider configuration or this create's recorded
|
|
377
|
+
// region may choose where new project resources live.
|
|
378
|
+
return createRegion ?? this.config.region ?? DEFAULT_REGION;
|
|
374
379
|
}
|
|
375
380
|
/**
|
|
376
381
|
* Whether each project gets its own Fly app on its own custom 6PN network.
|
|
@@ -480,10 +485,51 @@ class FlyioSandboxProvider {
|
|
|
480
485
|
* `<app>.internal` already resolves and Flycast would add nothing.
|
|
481
486
|
* @returns The declared services, or `undefined` when none are configured.
|
|
482
487
|
*/
|
|
483
|
-
privateServices() {
|
|
488
|
+
privateServices(createRegion) {
|
|
484
489
|
if (!this.appPerProject())
|
|
485
490
|
return undefined;
|
|
486
|
-
|
|
491
|
+
const fallback = this.config.privateServices ?? parsePrivateServices(process.env.FLY_SANDBOX_PRIVATE_SERVICES);
|
|
492
|
+
if (!createRegion || createRegion === DEFAULT_REGION)
|
|
493
|
+
return fallback;
|
|
494
|
+
const suffix = createRegion.replace(/-/g, '_').toUpperCase();
|
|
495
|
+
const variable = `FLY_SANDBOX_PRIVATE_SERVICES_${suffix}`;
|
|
496
|
+
const regional = this.config.privateServicesByRegion?.[createRegion] ??
|
|
497
|
+
parsePrivateServices(process.env[variable]);
|
|
498
|
+
if (!regional && fallback?.length) {
|
|
499
|
+
throw new Error(`No Fly private services are configured for create region "${createRegion}"; ` +
|
|
500
|
+
`set ${variable} (or config.privateServicesByRegion.${createRegion}). ` +
|
|
501
|
+
"Refusing to reuse another region's tenant route.");
|
|
502
|
+
}
|
|
503
|
+
return regional ?? fallback;
|
|
504
|
+
}
|
|
505
|
+
/** Fence and cancel every provisioning mutation at its owning attempt. */
|
|
506
|
+
async requestForAttempt(attempt, path, options) {
|
|
507
|
+
if (attempt.signal.aborted)
|
|
508
|
+
throw attempt.signal.reason;
|
|
509
|
+
await this.config.createAttemptJournal?.assertGeneration(attempt.attemptId, attempt.generation);
|
|
510
|
+
if (attempt.signal.aborted)
|
|
511
|
+
throw attempt.signal.reason;
|
|
512
|
+
const result = await this.client.request(path, { ...options, signal: attempt.signal });
|
|
513
|
+
// A response can arrive after ownership was lost. The write may remain
|
|
514
|
+
// unresolved, but a superseded worker must not journal/adopt its outcome.
|
|
515
|
+
if (attempt.signal.aborted)
|
|
516
|
+
throw attempt.signal.reason;
|
|
517
|
+
await this.config.createAttemptJournal?.assertGeneration(attempt.attemptId, attempt.generation);
|
|
518
|
+
return result;
|
|
519
|
+
}
|
|
520
|
+
/** Authorize a DELETE from the committed retired row on its live lock session. */
|
|
521
|
+
async deleteRetired(intent, generation, path, signal) {
|
|
522
|
+
const journal = this.config.createAttemptJournal;
|
|
523
|
+
if (!journal)
|
|
524
|
+
throw new Error('Retired deletion requires its durable create-attempt journal');
|
|
525
|
+
await journal.authorizeDestruction(intent.attemptId, generation);
|
|
526
|
+
return this.requestForDestruction(path, { method: 'DELETE', nullOn: [404] }, signal);
|
|
527
|
+
}
|
|
528
|
+
/** The sole request dispatch for retirement and settlement, always lock-fenced. */
|
|
529
|
+
async requestForDestruction(path, options, signal) {
|
|
530
|
+
if (signal.aborted)
|
|
531
|
+
throw signal.reason;
|
|
532
|
+
return this.client.request(path, { ...options, signal });
|
|
487
533
|
}
|
|
488
534
|
/**
|
|
489
535
|
* Idempotently ensures the Fly app exists, creating it on its own custom 6PN
|
|
@@ -505,27 +551,52 @@ class FlyioSandboxProvider {
|
|
|
505
551
|
* @returns `routes`: target app name → allocated private address, for
|
|
506
552
|
* recording in the Machine's metadata (empty when no private services are
|
|
507
553
|
* declared); `allocated`: the subset of `routes` this call created, which
|
|
508
|
-
* the caller releases when the boot fails
|
|
509
|
-
*
|
|
510
|
-
* held — only this call's own allocations are safe to take back).
|
|
554
|
+
* the caller releases when the boot fails. The app itself always survives:
|
|
555
|
+
* another create can adopt it by name while this call is still failing.
|
|
511
556
|
*/
|
|
512
557
|
async ensureApp(app, options = {}) {
|
|
513
558
|
const existing = await this.client.request(`/apps/${app}`, { nullOn: [404] });
|
|
514
559
|
let createdHere = false;
|
|
560
|
+
if (existing && options.requireCreated) {
|
|
561
|
+
return { routes: {}, allocated: {}, created: false };
|
|
562
|
+
}
|
|
515
563
|
if (!existing) {
|
|
516
564
|
const body = { name: app, org_slug: this.orgSlug() };
|
|
517
565
|
const network = this.networkFor(app);
|
|
518
566
|
if (network)
|
|
519
567
|
body.network = network;
|
|
520
568
|
let created = true;
|
|
569
|
+
const attempt = options.attempt;
|
|
570
|
+
const journal = this.config.createAttemptJournal;
|
|
571
|
+
if (attempt && journal) {
|
|
572
|
+
// Persist who is attempting the create BEFORE Fly can accept it. The
|
|
573
|
+
// separate accepted bit below prevents a pending/unknown POST from
|
|
574
|
+
// becoming deletion authority after a crash or reconciliation error.
|
|
575
|
+
await journal.recordAppProvenance(attempt.attemptId, attempt.generation, 'pending');
|
|
576
|
+
}
|
|
521
577
|
try {
|
|
522
|
-
|
|
578
|
+
if (attempt) {
|
|
579
|
+
await this.requestForAttempt(attempt, `/apps`, { method: 'POST', body });
|
|
580
|
+
}
|
|
581
|
+
else {
|
|
582
|
+
await this.client.request(`/apps`, { method: 'POST', body });
|
|
583
|
+
}
|
|
584
|
+
// Accepted POST provenance is established before later policy/route
|
|
585
|
+
// steps can fail, so diagnostic cleanup can still retire its own app.
|
|
586
|
+
if (options.ownership)
|
|
587
|
+
options.ownership.created = true;
|
|
588
|
+
if (attempt && journal) {
|
|
589
|
+
await journal.recordAppProvenance(attempt.attemptId, attempt.generation, 'created');
|
|
590
|
+
}
|
|
523
591
|
logger.info('Created Fly sandbox app', { app, network: body.network });
|
|
524
592
|
}
|
|
525
593
|
catch (error) {
|
|
526
594
|
// 409/422 means another concurrent boot created the same app first, which
|
|
527
595
|
// is exactly the outcome we wanted. Anything else is a real failure.
|
|
528
596
|
if (error instanceof FlyApiError && (error.status === 409 || error.status === 422)) {
|
|
597
|
+
if (attempt && journal) {
|
|
598
|
+
await journal.recordAppProvenance(attempt.attemptId, attempt.generation, 'adopted');
|
|
599
|
+
}
|
|
529
600
|
logger.debug('Fly sandbox app already exists', { app });
|
|
530
601
|
created = false;
|
|
531
602
|
}
|
|
@@ -535,45 +606,18 @@ class FlyioSandboxProvider {
|
|
|
535
606
|
}
|
|
536
607
|
createdHere = created;
|
|
537
608
|
}
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
await this.assignSharedIpv4(app);
|
|
541
|
-
await this.applyEgressPolicy(app);
|
|
542
|
-
if (options.privateRoutes === false)
|
|
543
|
-
return { routes: {}, allocated: {}, created: createdHere };
|
|
544
|
-
return {
|
|
545
|
-
...(await this.ensurePrivateRoutes(app, Boolean(existing))),
|
|
546
|
-
created: createdHere,
|
|
547
|
-
};
|
|
548
|
-
}
|
|
549
|
-
catch (error) {
|
|
550
|
-
if (createdHere)
|
|
551
|
-
await this.discardCreatedApp(app, {});
|
|
552
|
-
throw error;
|
|
609
|
+
if (options.requireCreated && !createdHere) {
|
|
610
|
+
return { routes: {}, allocated: {}, created: false };
|
|
553
611
|
}
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
* Deletes an app THIS call created, after a later step of the same create
|
|
557
|
-
* failed. Without it every failed create leaves an app behind with its
|
|
558
|
-
* addresses and volume: on 2026-09-28, 34 empty `mol-prod-*` apps had piled
|
|
559
|
-
* up in production, one per deploy that hit the organization's Machine limit
|
|
560
|
-
* (`POST /machines` → 422) after its app was already made. An app that
|
|
561
|
-
* existed before the call is never touched — it may hold a working Machine.
|
|
562
|
-
* @param app - The app this call created.
|
|
563
|
-
* @param privateRoutes - Flycast addresses already allocated for it.
|
|
564
|
-
*/
|
|
565
|
-
async discardCreatedApp(app, privateRoutes) {
|
|
566
|
-
try {
|
|
567
|
-
await this.client.request(`/apps/${app}`, { method: 'DELETE', nullOn: [404] });
|
|
568
|
-
logger.info('Deleted a Fly sandbox app whose create failed', { app });
|
|
612
|
+
if (createdHere && this.config.assignSharedIpv4) {
|
|
613
|
+
await this.assignSharedIpv4(app, options.attempt);
|
|
569
614
|
}
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
error,
|
|
574
|
-
});
|
|
615
|
+
await this.applyEgressPolicy(app, options.attempt, options.createRegion);
|
|
616
|
+
if (options.privateRoutes === false) {
|
|
617
|
+
return { routes: {}, allocated: {}, created: createdHere };
|
|
575
618
|
}
|
|
576
|
-
await this.
|
|
619
|
+
const routes = await this.ensurePrivateRoutes(app, Boolean(existing), options.attempt, options.createRegion);
|
|
620
|
+
return { ...routes, created: createdHere };
|
|
577
621
|
}
|
|
578
622
|
/**
|
|
579
623
|
* Allocates a Flycast private address on every declared service app, INTO this
|
|
@@ -604,37 +648,42 @@ class FlyioSandboxProvider {
|
|
|
604
648
|
* the only addresses safe to release when the boot fails after allocation,
|
|
605
649
|
* since the recorded ones belong to Machines an earlier boot may still run.
|
|
606
650
|
*/
|
|
607
|
-
async ensurePrivateRoutes(app, appExisted) {
|
|
608
|
-
const services = this.privateServices();
|
|
651
|
+
async ensurePrivateRoutes(app, appExisted, attempt, createRegion) {
|
|
652
|
+
const services = this.privateServices(createRegion);
|
|
609
653
|
const network = this.networkFor(app);
|
|
610
654
|
if (!services?.length || !network)
|
|
611
655
|
return { routes: {}, allocated: {} };
|
|
612
656
|
const routes = appExisted ? await this.recordedPrivateRoutes(app) : {};
|
|
613
657
|
// The addresses THIS call allocated, tracked apart from `routes` so a
|
|
614
|
-
// failure part-way down the next list can release exactly them.
|
|
615
|
-
//
|
|
616
|
-
//
|
|
617
|
-
// allocated for an earlier service stays behind on its target app, inside a
|
|
618
|
-
// 6PN whose app is gone, and Fly's listing does not report which network an
|
|
619
|
-
// address belongs to: nothing could ever find it there to release it.
|
|
658
|
+
// failure part-way down the next list can release exactly them. These
|
|
659
|
+
// addresses live on OTHER apps, so deleting this project's app would not
|
|
660
|
+
// release them; Fly's listing also cannot reconstruct their target network.
|
|
620
661
|
const allocated = {};
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
662
|
+
for (const service of services) {
|
|
663
|
+
if (routes[service.app])
|
|
664
|
+
continue;
|
|
665
|
+
if (attempt && this.config.createAttemptJournal) {
|
|
666
|
+
await this.config.createAttemptJournal.recordAddress(attempt.attemptId, {
|
|
667
|
+
targetApp: service.app,
|
|
668
|
+
network,
|
|
669
|
+
owned: false,
|
|
670
|
+
resolved: false,
|
|
671
|
+
});
|
|
672
|
+
}
|
|
673
|
+
const ip = await this.allocatePrivateAddress(service, network, attempt);
|
|
674
|
+
if (attempt && this.config.createAttemptJournal) {
|
|
675
|
+
await this.config.createAttemptJournal.recordAddress(attempt.attemptId, {
|
|
676
|
+
targetApp: service.app,
|
|
677
|
+
network,
|
|
678
|
+
...(ip ? { ip } : {}),
|
|
679
|
+
owned: Boolean(ip),
|
|
680
|
+
resolved: true,
|
|
681
|
+
});
|
|
682
|
+
}
|
|
683
|
+
if (ip) {
|
|
684
|
+
routes[service.app] = ip;
|
|
685
|
+
allocated[service.app] = ip;
|
|
630
686
|
}
|
|
631
|
-
}
|
|
632
|
-
catch (error) {
|
|
633
|
-
// Only what this call allocated. An address recorded by an EARLIER boot
|
|
634
|
-
// (`appExisted`) is left alone: the app survived this failure and may
|
|
635
|
-
// still hold a Machine reaching through that route.
|
|
636
|
-
await this.releasePrivateRoutes(allocated);
|
|
637
|
-
throw error;
|
|
638
687
|
}
|
|
639
688
|
return { routes, allocated };
|
|
640
689
|
}
|
|
@@ -667,50 +716,26 @@ class FlyioSandboxProvider {
|
|
|
667
716
|
return {};
|
|
668
717
|
}
|
|
669
718
|
/**
|
|
670
|
-
*
|
|
671
|
-
*
|
|
672
|
-
*
|
|
673
|
-
* retried queue job — leaves a Machine holding the app's single-attach volume.
|
|
674
|
-
* The next create's `POST /machines` then fails with `412 volume already
|
|
675
|
-
* claimed by machine <id>`, and the project can never recover on its own. The
|
|
676
|
-
* app is one-per-project and holds exactly one sandbox, so any managed Machine
|
|
677
|
-
* still here is a stale partial: destroy it (force covers running/suspended) to
|
|
678
|
-
* free the volume and make create idempotent per project.
|
|
679
|
-
* @param app - The Fly app name.
|
|
719
|
+
* Read and normalize Fly's assignment list once for allocation and settlement.
|
|
720
|
+
* Current Fly returns `{ips: [...]}` and a network object; the older array /
|
|
721
|
+
* `ip_assignments` shapes remain readable for compatibility with saved clients.
|
|
680
722
|
*/
|
|
681
|
-
async
|
|
682
|
-
const
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
return;
|
|
695
|
-
}
|
|
696
|
-
for (const machine of Array.isArray(machines) ? machines : []) {
|
|
697
|
-
if (machine.config?.metadata?.[managedKey] !== 'true')
|
|
698
|
-
continue;
|
|
699
|
-
try {
|
|
700
|
-
await this.client.request(`/apps/${app}/machines/${machine.id}?force=true`, {
|
|
701
|
-
method: 'DELETE',
|
|
702
|
-
nullOn: [404],
|
|
703
|
-
});
|
|
704
|
-
logger.info('Destroyed a stale Machine before re-creating the sandbox', {
|
|
705
|
-
app,
|
|
706
|
-
machineId: machine.id,
|
|
707
|
-
});
|
|
708
|
-
}
|
|
709
|
-
catch (error) {
|
|
710
|
-
logger.warn('Failed to destroy a stale Machine before create; create may fail with ' +
|
|
711
|
-
'412 volume already claimed', { app, machineId: machine.id, error });
|
|
723
|
+
async listPrivateAssignments(app, signal) {
|
|
724
|
+
const listed = signal
|
|
725
|
+
? await this.requestForDestruction(`/apps/${app}/ip_assignments`, {}, signal)
|
|
726
|
+
: await this.client.request(`/apps/${app}/ip_assignments`);
|
|
727
|
+
const assignments = Array.isArray(listed) ? listed : (listed?.ips ?? listed?.ip_assignments);
|
|
728
|
+
if (!assignments)
|
|
729
|
+
return null;
|
|
730
|
+
return assignments.map((assignment) => {
|
|
731
|
+
if (typeof assignment.network === 'string')
|
|
732
|
+
return assignment;
|
|
733
|
+
const network = assignment.network;
|
|
734
|
+
if (!network || network.org_slug !== this.orgSlug()) {
|
|
735
|
+
return { ...assignment, network: undefined };
|
|
712
736
|
}
|
|
713
|
-
|
|
737
|
+
return { ...assignment, network: network.name ?? '' };
|
|
738
|
+
});
|
|
714
739
|
}
|
|
715
740
|
/**
|
|
716
741
|
* Allocates ONE Flycast private address on a target app, for this project's
|
|
@@ -728,7 +753,7 @@ class FlyioSandboxProvider {
|
|
|
728
753
|
* address — an allocation this provider can neither verify nor release later
|
|
729
754
|
* is untracked state, and the boot is the last moment it can be reported.
|
|
730
755
|
*/
|
|
731
|
-
async allocatePrivateAddress(service, network) {
|
|
756
|
+
async allocatePrivateAddress(service, network, attempt) {
|
|
732
757
|
let assignment;
|
|
733
758
|
try {
|
|
734
759
|
// NO org_slug here. The ip_assignments endpoint derives the org from the
|
|
@@ -737,25 +762,28 @@ class FlyioSandboxProvider {
|
|
|
737
762
|
// slug that app creation REQUIRES is refused here; omitting it returns 200
|
|
738
763
|
// and allocates the address). app creation still sends org_slug — the two
|
|
739
764
|
// endpoints genuinely differ.
|
|
740
|
-
|
|
765
|
+
const request = {
|
|
741
766
|
method: 'POST',
|
|
742
767
|
body: {
|
|
743
768
|
type: this.config.privateIpAssignmentType ?? DEFAULT_PRIVATE_IP_TYPE,
|
|
744
769
|
network,
|
|
745
770
|
},
|
|
746
|
-
}
|
|
771
|
+
};
|
|
772
|
+
assignment = attempt
|
|
773
|
+
? await this.requestForAttempt(attempt, `/apps/${service.app}/ip_assignments`, request)
|
|
774
|
+
: await this.client.request(`/apps/${service.app}/ip_assignments`, request);
|
|
747
775
|
}
|
|
748
776
|
catch (error) {
|
|
749
|
-
// A conflict
|
|
750
|
-
//
|
|
751
|
-
//
|
|
752
|
-
// failure; it is an address this provider can no longer release, because
|
|
753
|
-
// Fly's listing does not report which network an address belongs to.
|
|
777
|
+
// A conflict can be the duplicate response to an accepted write whose
|
|
778
|
+
// first response was lost. Resolve it by target app AND network, then the
|
|
779
|
+
// caller journals the concrete IP for adoption or later release.
|
|
754
780
|
if (error instanceof FlyApiError && (error.status === 409 || error.status === 422)) {
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
781
|
+
const assignments = await this.listPrivateAssignments(service.app, attempt?.signal);
|
|
782
|
+
const existing = assignments?.find((candidate) => candidate.network === network);
|
|
783
|
+
if (existing?.ip)
|
|
784
|
+
return existing.ip;
|
|
785
|
+
throw new Error(`Fly rejected a private-address allocation for "${service.app}" on "${network}", ` +
|
|
786
|
+
'but no matching assignment could be resolved', { cause: error });
|
|
759
787
|
}
|
|
760
788
|
throw new Error(t('codeSandbox.flyio.error.privateRouteFailed', { app: service.app, network }, {
|
|
761
789
|
defaultValue: `Could not allocate a Flycast private address on Fly app "${service.app}" for ` +
|
|
@@ -781,28 +809,42 @@ class FlyioSandboxProvider {
|
|
|
781
809
|
});
|
|
782
810
|
return assignment.ip;
|
|
783
811
|
}
|
|
812
|
+
/** Validate the exact IP's current network; metadata alone is not DELETE authority. */
|
|
813
|
+
async privateRouteMatchesNetwork(target, ip, network, signal) {
|
|
814
|
+
if (!network)
|
|
815
|
+
return false;
|
|
816
|
+
const assignments = await this.listPrivateAssignments(target, signal);
|
|
817
|
+
return Boolean(assignments?.some((candidate) => candidate.ip === ip && candidate.network === network));
|
|
818
|
+
}
|
|
784
819
|
/**
|
|
785
|
-
* Releases
|
|
786
|
-
*
|
|
787
|
-
* Best-effort: the sandbox is being destroyed either way, and an address left
|
|
788
|
-
* behind is a reconciliation item (`fly ips list --app <target>`) rather than
|
|
789
|
-
* a failed teardown. It routes into a 6PN that no longer has any app in it.
|
|
790
|
-
* @param routes - Target app name → private address, from Machine metadata.
|
|
820
|
+
* Releases only Flycast addresses independently validated for this network.
|
|
821
|
+
* Metadata identifies candidates; it never authorizes a cross-network DELETE.
|
|
791
822
|
*/
|
|
792
|
-
async releasePrivateRoutes(routes) {
|
|
823
|
+
async releasePrivateRoutes(routes, signal, network) {
|
|
824
|
+
let released = true;
|
|
793
825
|
for (const [target, ip] of Object.entries(routes)) {
|
|
794
826
|
try {
|
|
795
|
-
await this.
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
827
|
+
if (!(await this.privateRouteMatchesNetwork(target, ip, network, signal))) {
|
|
828
|
+
released = false;
|
|
829
|
+
logger.warn('Kept a Fly private (Flycast) address with unvalidated network ownership', {
|
|
830
|
+
target,
|
|
831
|
+
ip,
|
|
832
|
+
network,
|
|
833
|
+
});
|
|
834
|
+
continue;
|
|
835
|
+
}
|
|
836
|
+
await this.requestForDestruction(`/apps/${target}/ip_assignments/${encodeURIComponent(ip)}`, { method: 'DELETE', nullOn: [404] }, signal);
|
|
799
837
|
logger.debug('Released a Fly private (Flycast) address', { target, ip });
|
|
800
838
|
}
|
|
801
839
|
catch (error) {
|
|
840
|
+
if (signal.aborted)
|
|
841
|
+
throw signal.reason;
|
|
842
|
+
released = false;
|
|
802
843
|
logger.warn('Failed to release a Fly private (Flycast) address — it will keep pointing at a 6PN ' +
|
|
803
844
|
'with no apps in it until it is reconciled with `fly ips list`', { target, ip, error });
|
|
804
845
|
}
|
|
805
846
|
}
|
|
847
|
+
return released;
|
|
806
848
|
}
|
|
807
849
|
/**
|
|
808
850
|
* Resolves the ports the egress network policy allows, or `undefined` when no
|
|
@@ -821,8 +863,8 @@ class FlyioSandboxProvider {
|
|
|
821
863
|
* about a rule with no ports, so accepting `[]` would ship a security
|
|
822
864
|
* control whose behaviour is a guess.
|
|
823
865
|
*/
|
|
824
|
-
egressAllowedPorts() {
|
|
825
|
-
return mergeEgressPorts(this.configuredEgressPorts(), this.privateServices());
|
|
866
|
+
egressAllowedPorts(createRegion) {
|
|
867
|
+
return mergeEgressPorts(this.configuredEgressPorts(), this.privateServices(createRegion));
|
|
826
868
|
}
|
|
827
869
|
/**
|
|
828
870
|
* Resolves ONLY the ports the operator configured, before the declared private
|
|
@@ -855,8 +897,8 @@ class FlyioSandboxProvider {
|
|
|
855
897
|
* @param app - The Fly app name.
|
|
856
898
|
* @returns Nothing; resolves once the policy is in place (or none was configured).
|
|
857
899
|
*/
|
|
858
|
-
async applyEgressPolicy(app) {
|
|
859
|
-
const ports = this.egressAllowedPorts();
|
|
900
|
+
async applyEgressPolicy(app, attempt, createRegion) {
|
|
901
|
+
const ports = this.egressAllowedPorts(createRegion);
|
|
860
902
|
if (!ports)
|
|
861
903
|
return;
|
|
862
904
|
const name = this.config.egressPolicyName ?? DEFAULT_EGRESS_POLICY_NAME;
|
|
@@ -888,7 +930,18 @@ class FlyioSandboxProvider {
|
|
|
888
930
|
selector: { all: true },
|
|
889
931
|
rules: [{ action: 'allow', direction: 'egress', ports }],
|
|
890
932
|
};
|
|
891
|
-
|
|
933
|
+
if (attempt) {
|
|
934
|
+
await this.requestForAttempt(attempt, `/apps/${app}/network_policies`, {
|
|
935
|
+
method: 'POST',
|
|
936
|
+
body: policy,
|
|
937
|
+
});
|
|
938
|
+
}
|
|
939
|
+
else {
|
|
940
|
+
await this.client.request(`/apps/${app}/network_policies`, {
|
|
941
|
+
method: 'POST',
|
|
942
|
+
body: policy,
|
|
943
|
+
});
|
|
944
|
+
}
|
|
892
945
|
logger.info('Applied Fly egress network policy', {
|
|
893
946
|
app,
|
|
894
947
|
name,
|
|
@@ -966,7 +1019,7 @@ class FlyioSandboxProvider {
|
|
|
966
1019
|
* rather than failing the boot.
|
|
967
1020
|
* @param app - The Fly app name.
|
|
968
1021
|
*/
|
|
969
|
-
async assignSharedIpv4(app) {
|
|
1022
|
+
async assignSharedIpv4(app, attempt) {
|
|
970
1023
|
try {
|
|
971
1024
|
// ONLY `type` — verified against the live API 2026-08-09. Both `region`
|
|
972
1025
|
// ("region selection is not supported for this IP type", 400) and
|
|
@@ -974,37 +1027,51 @@ class FlyioSandboxProvider {
|
|
|
974
1027
|
// sandbox app WITHOUT a public IP, so its preview URL never routed and the
|
|
975
1028
|
// IDE hung on "Loading preview…" forever. The org derives from the app and
|
|
976
1029
|
// a shared v4 is global (no region).
|
|
977
|
-
|
|
1030
|
+
const request = {
|
|
978
1031
|
method: 'POST',
|
|
979
|
-
body: {
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
1032
|
+
body: { type: this.config.ipAssignmentType ?? 'shared_v4' },
|
|
1033
|
+
};
|
|
1034
|
+
if (attempt)
|
|
1035
|
+
await this.requestForAttempt(attempt, `/apps/${app}/ip_assignments`, request);
|
|
1036
|
+
else
|
|
1037
|
+
await this.client.request(`/apps/${app}/ip_assignments`, request);
|
|
983
1038
|
}
|
|
984
1039
|
catch (error) {
|
|
1040
|
+
if (attempt?.signal.aborted)
|
|
1041
|
+
throw attempt.signal.reason;
|
|
1042
|
+
if (attempt) {
|
|
1043
|
+
await this.config.createAttemptJournal?.assertGeneration(attempt.attemptId, attempt.generation);
|
|
1044
|
+
}
|
|
985
1045
|
logger.warn('Failed to assign a shared IPv4 to the Fly sandbox app — the public preview URL will ' +
|
|
986
1046
|
'not resolve until an address is allocated', { app, error });
|
|
987
1047
|
}
|
|
988
1048
|
}
|
|
989
1049
|
/**
|
|
990
|
-
*
|
|
991
|
-
*
|
|
992
|
-
*
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
*
|
|
1000
|
-
* @
|
|
1050
|
+
* Return the readable, sanitized identity shared by attempts for one caller label.
|
|
1051
|
+
* @param name - The caller's volume label.
|
|
1052
|
+
* @returns The `[a-z0-9_]` stem used by {@link volumeName}.
|
|
1053
|
+
*/
|
|
1054
|
+
volumeNameStem(name) {
|
|
1055
|
+
const sanitized = name.toLowerCase().replace(/[^a-z0-9_]/g, '_');
|
|
1056
|
+
return `${sanitized.slice(0, 17)}_`;
|
|
1057
|
+
}
|
|
1058
|
+
/**
|
|
1059
|
+
* Give one create attempt a unique Fly volume name (Fly permits duplicate names).
|
|
1060
|
+
* @param name - The caller's volume label.
|
|
1061
|
+
* @param attemptId - The Machine create attempt's identity.
|
|
1062
|
+
* @returns A sanitized name of at most 30 characters, including the full attempt's digest.
|
|
1001
1063
|
*/
|
|
1002
|
-
volumeName(name) {
|
|
1064
|
+
volumeName(name, attemptId) {
|
|
1065
|
+
const digest = createHash('sha256').update(`${name}\0${attemptId}`).digest('hex').slice(0, 12);
|
|
1066
|
+
return `${this.volumeNameStem(name)}${digest}`;
|
|
1067
|
+
}
|
|
1068
|
+
/** The stable pre-attempt name under which durable workspaces were created. */
|
|
1069
|
+
workspaceVolumeName(name) {
|
|
1003
1070
|
const sanitized = name.toLowerCase().replace(/[^a-z0-9_]/g, '_');
|
|
1004
1071
|
if (sanitized.length <= 30)
|
|
1005
1072
|
return sanitized;
|
|
1006
|
-
const
|
|
1007
|
-
return `${sanitized.slice(0, 17)}_${
|
|
1073
|
+
const digest = createHash('sha256').update(name).digest('hex').slice(0, 12);
|
|
1074
|
+
return `${sanitized.slice(0, 17)}_${digest}`;
|
|
1008
1075
|
}
|
|
1009
1076
|
/**
|
|
1010
1077
|
* Idempotently ensures a volume exists in the app and returns its id.
|
|
@@ -1014,28 +1081,138 @@ class FlyioSandboxProvider {
|
|
|
1014
1081
|
* size are all known — rather than through the core's optional
|
|
1015
1082
|
* `createVolume(name)`; see the module `@remarks`.
|
|
1016
1083
|
* @param app - The Fly app name.
|
|
1017
|
-
* @param
|
|
1084
|
+
* @param flyName - This create attempt's unique, already-sanitized volume name.
|
|
1018
1085
|
* @param sizeGB - Size to create the volume with, in whole GB.
|
|
1019
|
-
* @
|
|
1086
|
+
* @param compute - The exact guest sent with the Machine create.
|
|
1087
|
+
* @returns The Fly volume id and whether this call created the volume.
|
|
1020
1088
|
*/
|
|
1021
|
-
async ensureVolume(app, name, sizeGB) {
|
|
1022
|
-
const
|
|
1023
|
-
const
|
|
1024
|
-
if (
|
|
1025
|
-
|
|
1026
|
-
|
|
1089
|
+
async ensureVolume(app, projectId, name, attempt, sizeGB, compute, region) {
|
|
1090
|
+
const journal = this.config.createAttemptJournal;
|
|
1091
|
+
const bound = await journal?.findWorkspace(projectId, name);
|
|
1092
|
+
if (bound?.id) {
|
|
1093
|
+
const volumes = await this.client.request(`/apps/${app}/volumes`, {
|
|
1094
|
+
signal: attempt.signal,
|
|
1095
|
+
});
|
|
1096
|
+
const existing = (volumes ?? []).find((volume) => volume.id === bound.id && volume.state !== 'destroyed' && volume.state !== 'destroying');
|
|
1097
|
+
if (!existing) {
|
|
1098
|
+
throw new Error(`Durable workspace volume "${bound.id}" cannot be read; refusing to replace it`);
|
|
1099
|
+
}
|
|
1100
|
+
await journal?.recordVolume(attempt.attemptId, {
|
|
1101
|
+
...bound,
|
|
1102
|
+
id: existing.id,
|
|
1103
|
+
logicalName: name,
|
|
1104
|
+
workspace: true,
|
|
1105
|
+
resolved: true,
|
|
1106
|
+
createdByAttemptId: undefined,
|
|
1107
|
+
adoptedByAttemptId: attempt.attemptId,
|
|
1108
|
+
});
|
|
1109
|
+
return { id: existing.id, name: bound.name, workspace: true };
|
|
1110
|
+
}
|
|
1111
|
+
// Pre-attempt releases used a deterministic name. Adopt that legacy volume
|
|
1112
|
+
// by ID once; all later recreation uses the durable binding above.
|
|
1113
|
+
const durableName = this.workspaceVolumeName(name);
|
|
1114
|
+
const existing = await this.findVolume(app, durableName, attempt.signal);
|
|
1115
|
+
if (existing) {
|
|
1116
|
+
await journal?.recordVolume(attempt.attemptId, {
|
|
1117
|
+
id: existing.id,
|
|
1118
|
+
name: durableName,
|
|
1119
|
+
logicalName: name,
|
|
1120
|
+
workspace: true,
|
|
1121
|
+
resolved: true,
|
|
1122
|
+
adoptedByAttemptId: attempt.attemptId,
|
|
1123
|
+
});
|
|
1124
|
+
return { id: existing.id, name: durableName, workspace: true };
|
|
1125
|
+
}
|
|
1126
|
+
const attemptName = this.volumeName(name, attempt.attemptId);
|
|
1127
|
+
await journal?.recordVolume(attempt.attemptId, {
|
|
1128
|
+
name: attemptName,
|
|
1129
|
+
logicalName: name,
|
|
1130
|
+
workspace: true,
|
|
1131
|
+
resolved: false,
|
|
1132
|
+
createdByAttemptId: attempt.attemptId,
|
|
1133
|
+
});
|
|
1134
|
+
try {
|
|
1135
|
+
const created = await this.createFreshVolume(app, attemptName, attempt, sizeGB, compute, region);
|
|
1136
|
+
await journal?.recordVolume(attempt.attemptId, {
|
|
1137
|
+
id: created.id,
|
|
1138
|
+
name: attemptName,
|
|
1139
|
+
logicalName: name,
|
|
1140
|
+
workspace: false,
|
|
1141
|
+
resolved: true,
|
|
1142
|
+
createdByAttemptId: created.accepted ? attempt.attemptId : undefined,
|
|
1143
|
+
adoptedByAttemptId: created.accepted ? undefined : attempt.attemptId,
|
|
1144
|
+
});
|
|
1145
|
+
return { id: created.id, name: attemptName, workspace: false };
|
|
1146
|
+
}
|
|
1147
|
+
catch (error) {
|
|
1148
|
+
if (error instanceof FlyApiError &&
|
|
1149
|
+
error.method.toUpperCase() === 'POST' &&
|
|
1150
|
+
error.path === `/apps/${app}/volumes` &&
|
|
1151
|
+
error.status >= 400 &&
|
|
1152
|
+
error.status < 500 &&
|
|
1153
|
+
![409, 422].includes(error.status)) {
|
|
1154
|
+
// A conflict/validation response can describe a write whose earlier
|
|
1155
|
+
// response was lost, so those outcomes stay unresolved. The 429 replay
|
|
1156
|
+
// contract treats rate-limit rejection as definite; other non-conflict
|
|
1157
|
+
// 4xx responses reject this unique attempt name before creating it.
|
|
1158
|
+
await journal?.recordVolume(attempt.attemptId, {
|
|
1159
|
+
name: attemptName,
|
|
1160
|
+
logicalName: name,
|
|
1161
|
+
workspace: false,
|
|
1162
|
+
resolved: true,
|
|
1163
|
+
});
|
|
1164
|
+
}
|
|
1165
|
+
throw error;
|
|
1166
|
+
}
|
|
1167
|
+
}
|
|
1168
|
+
/**
|
|
1169
|
+
* Creates a volume with the expected Machine's guest as its placement hint.
|
|
1170
|
+
* A 429 is safe to replay directly. After a transport failure or 5xx, bounded
|
|
1171
|
+
* lookups reconcile the attempt's unique name; if it stays invisible the create
|
|
1172
|
+
* fails rather than replaying an ambiguous POST that could duplicate the volume.
|
|
1173
|
+
* @param app - The Fly app name.
|
|
1174
|
+
* @param flyName - This create attempt's unique, already-sanitized volume name.
|
|
1175
|
+
* @param sizeGB - Size to create the volume with, in whole GB.
|
|
1176
|
+
* @param compute - The exact guest sent with the Machine create.
|
|
1177
|
+
* @returns The new Fly volume id.
|
|
1178
|
+
*/
|
|
1179
|
+
async createFreshVolume(app, flyName, attempt, sizeGB, compute, region) {
|
|
1180
|
+
let adopted = false;
|
|
1181
|
+
const created = await this.requestForAttempt(attempt, `/apps/${app}/volumes`, {
|
|
1027
1182
|
method: 'POST',
|
|
1028
1183
|
body: {
|
|
1029
1184
|
name: flyName,
|
|
1030
|
-
region
|
|
1185
|
+
region,
|
|
1031
1186
|
size_gb: sizeGB,
|
|
1032
1187
|
encrypted: true,
|
|
1188
|
+
compute,
|
|
1189
|
+
},
|
|
1190
|
+
reconcile: async (error) => {
|
|
1191
|
+
// A 429 is Fly's confirmation that this request was rejected. Every
|
|
1192
|
+
// other retryable outcome is ambiguous: never replay it until a few
|
|
1193
|
+
// name lookups have had a chance to find the volume Fly may have made.
|
|
1194
|
+
if (error.status !== 0 && error.status < 500)
|
|
1195
|
+
return undefined;
|
|
1196
|
+
for (let lookup = 1; lookup <= CREATE_RECONCILE_ATTEMPTS; lookup++) {
|
|
1197
|
+
const existing = await this.findVolume(app, flyName, attempt.signal);
|
|
1198
|
+
if (existing) {
|
|
1199
|
+
adopted = true;
|
|
1200
|
+
return existing;
|
|
1201
|
+
}
|
|
1202
|
+
if (lookup < CREATE_RECONCILE_ATTEMPTS) {
|
|
1203
|
+
await this.client.sleep(retryDelayMs(lookup));
|
|
1204
|
+
}
|
|
1205
|
+
}
|
|
1206
|
+
// Fly permits duplicate volume names. Negative reads therefore cannot
|
|
1207
|
+
// authorize another POST: stop with the outcome unknown and let the
|
|
1208
|
+
// owning app's durable reconciler reclaim anything the write made.
|
|
1209
|
+
throw new Error(`Fly volume create "${flyName}" has an unknown outcome; refusing to replay it`, { cause: error });
|
|
1033
1210
|
},
|
|
1034
1211
|
});
|
|
1035
1212
|
if (!created?.id) {
|
|
1036
1213
|
throw new Error(t('codeSandbox.flyio.error.volumeCreateFailed', { name: flyName }, { defaultValue: `Fly volume "${flyName}" was created but the API returned no id.` }));
|
|
1037
1214
|
}
|
|
1038
|
-
return created.id;
|
|
1215
|
+
return { id: created.id, accepted: !adopted };
|
|
1039
1216
|
}
|
|
1040
1217
|
/**
|
|
1041
1218
|
* Finds a volume in an app by name, ignoring ones already being destroyed.
|
|
@@ -1043,9 +1220,10 @@ class FlyioSandboxProvider {
|
|
|
1043
1220
|
* @param flyName - The sanitized Fly volume name.
|
|
1044
1221
|
* @returns The matching volume, or `null`.
|
|
1045
1222
|
*/
|
|
1046
|
-
async findVolume(app, flyName) {
|
|
1223
|
+
async findVolume(app, flyName, signal) {
|
|
1047
1224
|
const volumes = await this.client.request(`/apps/${app}/volumes`, {
|
|
1048
1225
|
nullOn: [404],
|
|
1226
|
+
signal,
|
|
1049
1227
|
});
|
|
1050
1228
|
if (!Array.isArray(volumes))
|
|
1051
1229
|
return null;
|
|
@@ -1073,7 +1251,21 @@ class FlyioSandboxProvider {
|
|
|
1073
1251
|
* @param config - Project id, image, env, volume name, template, labels and resource limits.
|
|
1074
1252
|
* @returns A `Sandbox` bound to the new Machine.
|
|
1075
1253
|
*/
|
|
1076
|
-
|
|
1254
|
+
/**
|
|
1255
|
+
* Return the collision-free app name for a complete caller/project identity.
|
|
1256
|
+
* @param projectId - The complete project or per-deploy identity.
|
|
1257
|
+
* @returns The DNS-safe, at-most-63-character app name.
|
|
1258
|
+
*/
|
|
1259
|
+
appNameForProject(projectId) {
|
|
1260
|
+
return appNameForProject(this.appPrefix(), projectId);
|
|
1261
|
+
}
|
|
1262
|
+
/**
|
|
1263
|
+
* Create, reconcile and boot one uniquely identified Fly Machine attempt.
|
|
1264
|
+
* Automatic cleanup deletes only a volume or exact Machine ID with this attempt's accepted-create provenance. Apps always remain KEEP for deliberate retirement; adopted, inherited, listed, older-writer and ambiguous resources are also KEPT.
|
|
1265
|
+
* @param config - The sandbox image, project, storage and resource configuration.
|
|
1266
|
+
* @returns The running sandbox.
|
|
1267
|
+
*/
|
|
1268
|
+
async create(config, signal = new AbortController().signal) {
|
|
1077
1269
|
// Resolved BEFORE anything is provisioned, so a bad template id costs no app,
|
|
1078
1270
|
// no volume and no Machine.
|
|
1079
1271
|
let manifest = null;
|
|
@@ -1093,49 +1285,60 @@ class FlyioSandboxProvider {
|
|
|
1093
1285
|
// is checked too — the host is unroutable however the value arrives, and
|
|
1094
1286
|
// that env is exactly where a control plane puts the values it refuses to
|
|
1095
1287
|
// leave in a Machine config.
|
|
1096
|
-
|
|
1288
|
+
const createRegion = this.region(config.region);
|
|
1289
|
+
assertPrivateRoutesForEnv({ ...config.env, ...config.selfDeliveredEnv }, this.privateServices(createRegion));
|
|
1097
1290
|
const app = this.resolveApp(config.projectId);
|
|
1098
|
-
const
|
|
1291
|
+
const createAttemptId = randomUUID();
|
|
1292
|
+
const journal = this.config.createAttemptJournal;
|
|
1293
|
+
// The SQL journal stays closed until pre-journal/provenance-ignoring writers
|
|
1294
|
+
// have drained. No durable row or Fly resource is created while it is closed.
|
|
1295
|
+
await journal?.assertCreateAllowed();
|
|
1296
|
+
await journal?.begin({
|
|
1297
|
+
attemptId: createAttemptId,
|
|
1298
|
+
projectId: config.projectId,
|
|
1299
|
+
ownerProjectId: String(config.labels?.['molecule.production'] ?? config.projectId),
|
|
1300
|
+
appName: app,
|
|
1301
|
+
});
|
|
1302
|
+
const attempt = { attemptId: createAttemptId, generation: 1, signal };
|
|
1099
1303
|
try {
|
|
1100
|
-
|
|
1304
|
+
const { routes: privateRoutes } = await this.ensureApp(app, {
|
|
1305
|
+
attempt,
|
|
1306
|
+
createRegion,
|
|
1307
|
+
});
|
|
1308
|
+
const sandbox = await this.createInApp(app, privateRoutes, config, manifest, attempt);
|
|
1309
|
+
if (signal.aborted)
|
|
1310
|
+
throw signal.reason;
|
|
1311
|
+
await journal?.assertGeneration(attempt.attemptId, attempt.generation);
|
|
1312
|
+
await journal?.markState(createAttemptId, 'serving', sandbox.id);
|
|
1313
|
+
return sandbox;
|
|
1101
1314
|
}
|
|
1102
1315
|
catch (error) {
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
// Machine that would have carried the metadata was never created, and
|
|
1113
|
-
// Fly's listing cannot say which network an address belongs to. Take
|
|
1114
|
-
// back exactly this call's own allocations — never the routes an
|
|
1115
|
-
// earlier boot recorded, whose Machines may still be running.
|
|
1116
|
-
await this.releasePrivateRoutes(allocated);
|
|
1117
|
-
}
|
|
1316
|
+
// Create has NO deletion path. The durable attempt is settled by the
|
|
1317
|
+
// separate reconciler after this failure (or after its abandonment window).
|
|
1318
|
+
await journal?.markState(createAttemptId, 'failed').catch((journalError) => {
|
|
1319
|
+
logger.warn('Could not mark a Fly create attempt failed', {
|
|
1320
|
+
app,
|
|
1321
|
+
createAttemptId,
|
|
1322
|
+
error: journalError,
|
|
1323
|
+
});
|
|
1324
|
+
});
|
|
1118
1325
|
throw error;
|
|
1119
1326
|
}
|
|
1120
1327
|
}
|
|
1121
1328
|
/**
|
|
1122
|
-
* The part of {@link create} that
|
|
1123
|
-
* Machines, provision the volume, create the Machine and wait for it.
|
|
1329
|
+
* The part of {@link create} that provisions the volume, creates the Machine and waits for it.
|
|
1124
1330
|
* @param app - The app.
|
|
1125
1331
|
* @param privateRoutes - Flycast addresses allocated for the app.
|
|
1126
1332
|
* @param config - The sandbox config.
|
|
1127
1333
|
* @param manifest - The template manifest, when creating from a template.
|
|
1128
1334
|
* @returns The running sandbox.
|
|
1129
1335
|
*/
|
|
1130
|
-
async createInApp(app, privateRoutes, config, manifest) {
|
|
1131
|
-
|
|
1132
|
-
// provisioning — otherwise it still holds the app's single-attach volume and
|
|
1133
|
-
// `POST /machines` fails with `412 volume already claimed`. Makes create
|
|
1134
|
-
// idempotent so a retried queue job (or a user re-clicking Start) recovers.
|
|
1135
|
-
await this.destroyStaleMachines(app);
|
|
1336
|
+
async createInApp(app, privateRoutes, config, manifest, attempt) {
|
|
1337
|
+
const createAttemptId = attempt.attemptId;
|
|
1136
1338
|
const metadata = {
|
|
1137
1339
|
[`${this.metadataPrefix}.projectId`]: config.projectId,
|
|
1138
1340
|
[`${this.metadataPrefix}.managed`]: 'true',
|
|
1341
|
+
[`${this.metadataPrefix}.createAttempt`]: createAttemptId,
|
|
1139
1342
|
// Recorded so `destroy()` can release exactly the addresses this project
|
|
1140
1343
|
// allocated. Fly's IP listing does not report which network an address
|
|
1141
1344
|
// belongs to, so nothing else can reconstruct this.
|
|
@@ -1181,34 +1384,112 @@ class FlyioSandboxProvider {
|
|
|
1181
1384
|
// the image's idle command and serving nothing.
|
|
1182
1385
|
if (config.command?.length)
|
|
1183
1386
|
machineConfig.init = { exec: [...config.command] };
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1387
|
+
const volumeName = config.volumeName;
|
|
1388
|
+
if (volumeName) {
|
|
1389
|
+
metadata[`${this.metadataPrefix}.volumeName`] = volumeName;
|
|
1390
|
+
const volumeSizeGB = config.resources?.diskMB
|
|
1187
1391
|
? Math.max(1, Math.ceil(config.resources.diskMB / 1024))
|
|
1188
1392
|
: (this.config.defaultVolumeGB ?? DEFAULT_VOLUME_GB);
|
|
1189
|
-
const
|
|
1393
|
+
const volume = await this.ensureVolume(app, config.projectId, volumeName, attempt, volumeSizeGB, machineConfig.guest ?? {}, this.region(config.region));
|
|
1190
1394
|
// The caller may scope the volume to a subtree (e.g. /workspace/<appDir>)
|
|
1191
1395
|
// so regenerable tooling (node_modules) stays on the fast image-backed
|
|
1192
1396
|
// rootfs and only durable state pays the volume's small-file IO cost.
|
|
1193
|
-
machineConfig.mounts = [{ volume:
|
|
1397
|
+
machineConfig.mounts = [{ volume: volume.id, path: config.volumeMountPath ?? WORKSPACE_PATH }];
|
|
1194
1398
|
}
|
|
1195
1399
|
const service = this.previewService(config.idleAction);
|
|
1196
1400
|
if (service)
|
|
1197
1401
|
machineConfig.services = [service];
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1402
|
+
// The POST intent must survive a crash before dispatch. A NULL outcome on
|
|
1403
|
+
// an older row is unknown too; only a new not-started/rejected row proves
|
|
1404
|
+
// that no accepted Machine identity can be hidden by a later empty list.
|
|
1405
|
+
await this.config.createAttemptJournal?.recordMachineCreateOutcome(createAttemptId, attempt.generation, 'pending');
|
|
1406
|
+
let machine;
|
|
1407
|
+
let machineCreateResponseLost = false;
|
|
1408
|
+
try {
|
|
1409
|
+
machine = await this.requestForAttempt(attempt, `/apps/${app}/machines`, {
|
|
1410
|
+
method: 'POST',
|
|
1411
|
+
body: {
|
|
1412
|
+
name: createAttemptId,
|
|
1413
|
+
region: this.region(config.region),
|
|
1414
|
+
config: machineConfig,
|
|
1415
|
+
},
|
|
1416
|
+
// A just-created app propagates to the Machines API asynchronously, so this
|
|
1417
|
+
// POST can 404 "app not found" for a few seconds. Retry the 404 (safe: no
|
|
1418
|
+
// Machine is created when the app is not found) across the client's backoff.
|
|
1419
|
+
attempts: MACHINE_CREATE_ATTEMPTS,
|
|
1420
|
+
retryStatuses: MACHINE_CREATE_RETRY_STATUSES,
|
|
1421
|
+
// A 412 is transient only for the specific released-volume-claim lag.
|
|
1422
|
+
// Placement capacity must reach the caller's bounded retry at once,
|
|
1423
|
+
// with the volume preserved; every other 412 remains a real rejection.
|
|
1424
|
+
retryWhen: (error) => {
|
|
1425
|
+
if (error.status === 400)
|
|
1426
|
+
return /volume not found/i.test(error.body);
|
|
1427
|
+
if (error.status !== 412)
|
|
1428
|
+
return true;
|
|
1429
|
+
return (/volume already claimed by machine/i.test(error.body) ||
|
|
1430
|
+
/volume_placement_capacity/i.test(error.body));
|
|
1431
|
+
},
|
|
1432
|
+
reconcile: async (error) => {
|
|
1433
|
+
// Rate limiting and explicit precondition rejections did not create a
|
|
1434
|
+
// Machine, so their normal bounded replay is safe. A transport/5xx
|
|
1435
|
+
// result is ambiguous: reconcile ONLY this attempt's recorded identity
|
|
1436
|
+
// and never issue another create when the reads cannot find it.
|
|
1437
|
+
if (error.status === 429 || (error.status >= 400 && error.status < 500)) {
|
|
1438
|
+
return undefined;
|
|
1439
|
+
}
|
|
1440
|
+
// A Machine recovered by listing after a lost response is adoption
|
|
1441
|
+
// evidence, not an accepted POST response and never delete authority.
|
|
1442
|
+
machineCreateResponseLost = true;
|
|
1443
|
+
await this.config.createAttemptJournal?.recordMachineCreateOutcome(createAttemptId, attempt.generation, 'unknown');
|
|
1444
|
+
for (let lookup = 1; lookup <= CREATE_RECONCILE_ATTEMPTS; lookup++) {
|
|
1445
|
+
const machines = await this.client.request(`/apps/${app}/machines`, {
|
|
1446
|
+
nullOn: [404],
|
|
1447
|
+
signal: attempt.signal,
|
|
1448
|
+
});
|
|
1449
|
+
const existing = (machines ?? []).find((candidate) => candidate.name === createAttemptId ||
|
|
1450
|
+
candidate.config?.metadata?.[`${this.metadataPrefix}.createAttempt`] ===
|
|
1451
|
+
createAttemptId);
|
|
1452
|
+
if (existing) {
|
|
1453
|
+
await this.config.createAttemptJournal?.recordMachineObservation(createAttemptId, attempt.generation, existing.id);
|
|
1454
|
+
return existing;
|
|
1455
|
+
}
|
|
1456
|
+
if (lookup < CREATE_RECONCILE_ATTEMPTS) {
|
|
1457
|
+
await this.client.sleep(retryDelayMs(lookup));
|
|
1458
|
+
}
|
|
1459
|
+
}
|
|
1460
|
+
throw new Error(`Fly Machine create "${createAttemptId}" in app "${app}" has an unknown outcome; refusing to replay it`, { cause: error });
|
|
1461
|
+
},
|
|
1462
|
+
});
|
|
1463
|
+
}
|
|
1464
|
+
catch (error) {
|
|
1465
|
+
if (attempt.signal.aborted)
|
|
1466
|
+
throw attempt.signal.reason;
|
|
1467
|
+
if (error instanceof FlyApiError &&
|
|
1468
|
+
error.method.toUpperCase() === 'POST' &&
|
|
1469
|
+
error.path === `/apps/${app}/machines` &&
|
|
1470
|
+
error.status >= 400 &&
|
|
1471
|
+
error.status < 500) {
|
|
1472
|
+
try {
|
|
1473
|
+
await this.config.createAttemptJournal?.markMachineCreateFailed(createAttemptId, attempt.generation);
|
|
1474
|
+
}
|
|
1475
|
+
catch (journalError) {
|
|
1476
|
+
logger.warn('Could not record a definitive Fly Machine-create rejection', {
|
|
1477
|
+
app,
|
|
1478
|
+
createAttemptId,
|
|
1479
|
+
error: journalError,
|
|
1480
|
+
});
|
|
1481
|
+
}
|
|
1482
|
+
}
|
|
1483
|
+
throw error;
|
|
1484
|
+
}
|
|
1207
1485
|
if (!machine?.id) {
|
|
1208
1486
|
throw new Error(t('codeSandbox.flyio.error.createFailed', { app }, { defaultValue: `Fly Machine creation in app "${app}" returned no Machine id.` }));
|
|
1209
1487
|
}
|
|
1488
|
+
if (!machineCreateResponseLost) {
|
|
1489
|
+
await this.config.createAttemptJournal?.recordMachineProvenance(createAttemptId, attempt.generation, machine.id);
|
|
1490
|
+
}
|
|
1210
1491
|
if (manifest && config.templateId) {
|
|
1211
|
-
await this.restoreTemplate(app, machine.id, config.templateId, manifest);
|
|
1492
|
+
await this.restoreTemplate(app, machine.id, config.templateId, manifest, attempt);
|
|
1212
1493
|
// The restore waited for `started` and ran a command in it, so the state
|
|
1213
1494
|
// read back from the creation response is stale by definition.
|
|
1214
1495
|
return this.buildSandbox(app, machine.id, 'running');
|
|
@@ -1219,35 +1500,7 @@ class FlyioSandboxProvider {
|
|
|
1219
1500
|
// `412 failed_precondition: machine not running` until the Machine is up.
|
|
1220
1501
|
// Wait for `started` here so `create` always hands back a live sandbox, as
|
|
1221
1502
|
// the JSDoc promises — the template path already does this via restore.
|
|
1222
|
-
|
|
1223
|
-
await this.waitForStarted(app, machine.id);
|
|
1224
|
-
}
|
|
1225
|
-
catch (error) {
|
|
1226
|
-
// The boot failed after the Machine existed. This call created it and
|
|
1227
|
-
// handed it to NOBODY — the caller got a rejection, not a handle — so
|
|
1228
|
-
// leaving it behind strands state nothing else ever cleans up: a Machine
|
|
1229
|
-
// that reached `started` bills compute until it is reaped, and one that
|
|
1230
|
-
// did not keeps the app's single-attach volume claimed (the next
|
|
1231
|
-
// create's stale-Machine sweep only runs if this project is retried).
|
|
1232
|
-
// The created-app failure path below cannot do this either: it deletes
|
|
1233
|
-
// the APP, and `discardCreatedApp` deliberately has no Machine-delete
|
|
1234
|
-
// step — an app delete that runs under a live Machine either fails or
|
|
1235
|
-
// hides the Machine. Destroy the Machine first (the same order
|
|
1236
|
-
// `restoreTemplate`'s failure path and `destroy()` use), then rethrow
|
|
1237
|
-
// the boot failure. Best-effort: the boot failure is what the caller
|
|
1238
|
-
// must see; a failed cleanup is logged like every other cleanup here.
|
|
1239
|
-
try {
|
|
1240
|
-
await this.client.request(`/apps/${app}/machines/${machine.id}?force=true`, {
|
|
1241
|
-
method: 'DELETE',
|
|
1242
|
-
nullOn: [404],
|
|
1243
|
-
});
|
|
1244
|
-
}
|
|
1245
|
-
catch (cleanupError) {
|
|
1246
|
-
logger.warn('Failed to destroy the Fly Machine whose boot did not reach started — it will keep ' +
|
|
1247
|
-
'billing (and holding its volume) until it is reaped', { app, machineId: machine.id, error: cleanupError });
|
|
1248
|
-
}
|
|
1249
|
-
throw error;
|
|
1250
|
-
}
|
|
1503
|
+
await this.waitForStarted(app, machine.id, attempt);
|
|
1251
1504
|
return this.buildSandbox(app, machine.id, 'running');
|
|
1252
1505
|
}
|
|
1253
1506
|
/**
|
|
@@ -1486,8 +1739,10 @@ class FlyioSandboxProvider {
|
|
|
1486
1739
|
'TRUNCATED, so some sandboxes would be missing from the result.',
|
|
1487
1740
|
}));
|
|
1488
1741
|
}
|
|
1489
|
-
const
|
|
1490
|
-
const owned = apps
|
|
1742
|
+
const namespace = appNameNamespace(this.appPrefix());
|
|
1743
|
+
const owned = apps
|
|
1744
|
+
.map((app) => app.name)
|
|
1745
|
+
.filter((name) => namespace.searchPrefixes.some((prefix) => name?.startsWith(prefix)));
|
|
1491
1746
|
if (owned.length > FIND_MAX_APPS) {
|
|
1492
1747
|
throw new Error(t('codeSandbox.flyio.error.findTooManyApps', { count: String(owned.length), max: String(FIND_MAX_APPS) }, {
|
|
1493
1748
|
defaultValue: `Refusing to enumerate ${owned.length} Fly sandbox apps (cap ${FIND_MAX_APPS}): ` +
|
|
@@ -1497,6 +1752,834 @@ class FlyioSandboxProvider {
|
|
|
1497
1752
|
}
|
|
1498
1753
|
return owned;
|
|
1499
1754
|
}
|
|
1755
|
+
/** Read an existing app's live state; a list 404 after the app read is an error, not absence. */
|
|
1756
|
+
async readAttemptAppState(app, signal) {
|
|
1757
|
+
const [machines, volumes] = await Promise.all([
|
|
1758
|
+
this.requestForDestruction(`/apps/${app}/machines`, { nullOn: [404] }, signal),
|
|
1759
|
+
this.requestForDestruction(`/apps/${app}/volumes`, { nullOn: [404] }, signal),
|
|
1760
|
+
]);
|
|
1761
|
+
if (Array.isArray(machines) && Array.isArray(volumes)) {
|
|
1762
|
+
return {
|
|
1763
|
+
appGone: false,
|
|
1764
|
+
machines,
|
|
1765
|
+
volumes: volumes.filter((volume) => volume.state !== 'destroyed' && volume.state !== 'destroying'),
|
|
1766
|
+
};
|
|
1767
|
+
}
|
|
1768
|
+
// A Machines/Volumes-list 404 is not proof that a newly-created app is
|
|
1769
|
+
// gone. Confirm the app itself; if it exists, the list read was transient
|
|
1770
|
+
// and responsibility must remain durable.
|
|
1771
|
+
const existing = await this.requestForDestruction(`/apps/${app}`, { nullOn: [404] }, signal);
|
|
1772
|
+
if (existing)
|
|
1773
|
+
throw new Error(`Fly app "${app}" exists but its live-state listing failed`);
|
|
1774
|
+
return { appGone: true, machines: [], volumes: [] };
|
|
1775
|
+
}
|
|
1776
|
+
/** Prove a definitively rejected Machine create has no live consumer before retiring it. */
|
|
1777
|
+
async assertFailedMachineSweepable(intent, signal) {
|
|
1778
|
+
if ((intent.appCreatedByAttemptId === intent.attemptId && intent.appCreateResolved !== true) ||
|
|
1779
|
+
intent.volumes.some((volume) => volume.resolved !== true) ||
|
|
1780
|
+
intent.addresses.some((address) => !address.resolved)) {
|
|
1781
|
+
throw new Error(`Fly create attempt "${intent.attemptId}" still has unresolved ownership`);
|
|
1782
|
+
}
|
|
1783
|
+
const live = await this.readAttemptAppState(intent.appName, signal);
|
|
1784
|
+
if (signal.aborted)
|
|
1785
|
+
throw signal.reason;
|
|
1786
|
+
// Siblings matter only through an actual resource reference below, or at
|
|
1787
|
+
// the later app-deletion boundary. Their mere existence must not strand a
|
|
1788
|
+
// detached, proven-created volume in a shared app.
|
|
1789
|
+
if (live.appGone)
|
|
1790
|
+
return;
|
|
1791
|
+
if (this.appPerProject() && live.machines.length) {
|
|
1792
|
+
throw new Error(`Fly app "${intent.appName}" still has Machines; failed-create sweep is KEEP`);
|
|
1793
|
+
}
|
|
1794
|
+
const volumeIds = new Set(intent.volumes.map((volume) => volume.id).filter((id) => Boolean(id)));
|
|
1795
|
+
for (const machine of live.machines) {
|
|
1796
|
+
if (machine.config?.mounts?.some((mount) => mount.volume && volumeIds.has(mount.volume))) {
|
|
1797
|
+
throw new Error(`Fly Machine "${machine.id}" still references a failed-create volume; sweep is KEEP`);
|
|
1798
|
+
}
|
|
1799
|
+
}
|
|
1800
|
+
for (const volume of live.volumes) {
|
|
1801
|
+
if (volumeIds.has(volume.id) && volume.attached_machine_id) {
|
|
1802
|
+
throw new Error(`Fly volume "${volume.id}" is still attached to "${volume.attached_machine_id}"; sweep is KEEP`);
|
|
1803
|
+
}
|
|
1804
|
+
}
|
|
1805
|
+
}
|
|
1806
|
+
/** Exact durable Machine identity; unlike accepted provenance, sandboxId may be observational. */
|
|
1807
|
+
recordedMachineId(intent) {
|
|
1808
|
+
if (intent.machineId)
|
|
1809
|
+
return intent.machineId;
|
|
1810
|
+
const prefix = `${intent.appName}:`;
|
|
1811
|
+
if (!intent.sandboxId?.startsWith(prefix))
|
|
1812
|
+
return undefined;
|
|
1813
|
+
return intent.sandboxId.slice(prefix.length) || undefined;
|
|
1814
|
+
}
|
|
1815
|
+
/** True when no exact durable Machine ID can settle a possibly-dispatched POST. */
|
|
1816
|
+
hasUnresolvedMachineIdentity(intent) {
|
|
1817
|
+
if (this.recordedMachineId(intent))
|
|
1818
|
+
return false;
|
|
1819
|
+
const legacyUnknown = intent.machineCreateOutcome === undefined &&
|
|
1820
|
+
intent.machineCreateFailed !== true &&
|
|
1821
|
+
!intent.machineId &&
|
|
1822
|
+
!intent.sandboxId &&
|
|
1823
|
+
!intent.machineAdopted;
|
|
1824
|
+
return (legacyUnknown ||
|
|
1825
|
+
intent.machineCreateOutcome === 'pending' ||
|
|
1826
|
+
intent.machineCreateOutcome === 'unknown' ||
|
|
1827
|
+
intent.machineCreateOutcome === 'accepted' ||
|
|
1828
|
+
Boolean(intent.machineId || intent.sandboxId || intent.machineAdopted));
|
|
1829
|
+
}
|
|
1830
|
+
/**
|
|
1831
|
+
* The sole forgetting guard. A listing is never an absence proof: every
|
|
1832
|
+
* durable Machine/volume ID needs this pass's authorized DELETE success/404
|
|
1833
|
+
* or its own exact GET 404. Unresolved identities remain visible as KEEP.
|
|
1834
|
+
*/
|
|
1835
|
+
async assertAuthoritativeForget(intent, signal, settled, app) {
|
|
1836
|
+
// A successful non-forced app DELETE (or exact app 404) is authoritative
|
|
1837
|
+
// for every resource contained by that app. External addresses still need
|
|
1838
|
+
// their own settled records below.
|
|
1839
|
+
const containerSettled = app.settled;
|
|
1840
|
+
const machineId = this.recordedMachineId(intent);
|
|
1841
|
+
if (!containerSettled && this.hasUnresolvedMachineIdentity(intent)) {
|
|
1842
|
+
throw new Error(`Fly Machine identity for attempt "${intent.attemptId}" is unresolved; forgetting is KEEP`);
|
|
1843
|
+
}
|
|
1844
|
+
if (!containerSettled && machineId && !settled.machineIds.has(machineId)) {
|
|
1845
|
+
const machine = await this.requestForDestruction(`/apps/${intent.appName}/machines/${machineId}`, { nullOn: [404] }, signal);
|
|
1846
|
+
if (machine) {
|
|
1847
|
+
throw new Error(`Fly Machine "${machineId}" for attempt "${intent.attemptId}" still exists; forgetting is KEEP`);
|
|
1848
|
+
}
|
|
1849
|
+
}
|
|
1850
|
+
for (const volume of containerSettled ? [] : intent.volumes) {
|
|
1851
|
+
if (volume.resolved !== true) {
|
|
1852
|
+
throw new Error(`Fly volume "${volume.id ?? volume.name}" for attempt "${intent.attemptId}" is unresolved; forgetting is KEEP`);
|
|
1853
|
+
}
|
|
1854
|
+
if (!volume.id || settled.volumeIds.has(volume.id))
|
|
1855
|
+
continue;
|
|
1856
|
+
const live = await this.requestForDestruction(`/apps/${intent.appName}/volumes/${volume.id}`, { nullOn: [404] }, signal);
|
|
1857
|
+
if (live) {
|
|
1858
|
+
throw new Error(`Fly volume "${volume.id}" for attempt "${intent.attemptId}" still exists; forgetting is KEEP`);
|
|
1859
|
+
}
|
|
1860
|
+
}
|
|
1861
|
+
if (intent.addresses.some((address) => !address.resolved || address.owned)) {
|
|
1862
|
+
throw new Error(`Fly addresses for attempt "${intent.attemptId}" are not settled; KEEP`);
|
|
1863
|
+
}
|
|
1864
|
+
if (app.required && !app.settled) {
|
|
1865
|
+
throw new Error(`Fly app "${intent.appName}" has no authoritative absence result; KEEP`);
|
|
1866
|
+
}
|
|
1867
|
+
}
|
|
1868
|
+
/**
|
|
1869
|
+
* The ONE automatic deletion/settlement function. Its four invariants are:
|
|
1870
|
+
* (a) it runs only under the owning attempt's lifecycle lock, and re-reads the
|
|
1871
|
+
* durable record and Fly live state inside that lock before EVERY delete;
|
|
1872
|
+
* (b) automatic deletion additionally requires this attempt's accepted-POST
|
|
1873
|
+
* provenance and exact resource ID; adoption is never deletion authority;
|
|
1874
|
+
* (c) unknown outcomes and transient absence retain responsibility. The one
|
|
1875
|
+
* forgetting boundary below requires an authoritative exact-ID result for
|
|
1876
|
+
* every resource: an exact GET 404, or an authorized DELETE success/404. A
|
|
1877
|
+
* listing, empty listing, NULL ID or "no recorded resources" proves no
|
|
1878
|
+
* absence; an unresolved identity keeps the row and reports KEEP. Durable
|
|
1879
|
+
* addresses are released before app deletion/forgetting; and (d) cancellation
|
|
1880
|
+
* plus the attempt's current generation fences every Fly mutation. A stale,
|
|
1881
|
+
* serving or unreadable attempt therefore cannot delete anything or be forgotten.
|
|
1882
|
+
*/
|
|
1883
|
+
async settleCreateAttempt(snapshot, signal, generation, options = {}) {
|
|
1884
|
+
const automatic = options.automatic ?? false;
|
|
1885
|
+
const retire = options.retire ?? false;
|
|
1886
|
+
const failedMachineSweep = options.failedMachineSweep ?? false;
|
|
1887
|
+
const deleteApp = options.deleteApp ?? true;
|
|
1888
|
+
const forget = options.forget ?? true;
|
|
1889
|
+
const appRequired = deleteApp && this.appPerProject();
|
|
1890
|
+
// Automatic reconciliation never deletes an app: a container DELETE can
|
|
1891
|
+
// cascade into any sibling's retained or not-yet-resolved user resources.
|
|
1892
|
+
// An existing app remains a durable KEEP for deliberate retirement.
|
|
1893
|
+
const mayDeleteApp = appRequired && !automatic;
|
|
1894
|
+
const journal = this.config.createAttemptJournal;
|
|
1895
|
+
if (!journal)
|
|
1896
|
+
throw new Error('A durable create-attempt journal is required for settlement');
|
|
1897
|
+
const fresh = async () => {
|
|
1898
|
+
if (signal.aborted)
|
|
1899
|
+
throw signal.reason;
|
|
1900
|
+
await journal.assertGeneration(snapshot.attemptId, generation);
|
|
1901
|
+
const current = await journal.get(snapshot.attemptId);
|
|
1902
|
+
if (signal.aborted)
|
|
1903
|
+
throw signal.reason;
|
|
1904
|
+
if (!current || (current.generation ?? 1) !== generation) {
|
|
1905
|
+
throw new Error(`Fly create attempt "${snapshot.attemptId}" was superseded`);
|
|
1906
|
+
}
|
|
1907
|
+
if (current.state !== 'failed' && current.state !== 'abandoned') {
|
|
1908
|
+
throw new Error(`Fly create attempt "${snapshot.attemptId}" is no longer settleable`);
|
|
1909
|
+
}
|
|
1910
|
+
if ((options.expectedAppName && current.appName !== options.expectedAppName) ||
|
|
1911
|
+
(options.expectedMachineId && this.recordedMachineId(current) !== options.expectedMachineId)) {
|
|
1912
|
+
throw new Error(`Fly create attempt "${snapshot.attemptId}" changed outside the caller-validated scope`);
|
|
1913
|
+
}
|
|
1914
|
+
return current;
|
|
1915
|
+
};
|
|
1916
|
+
const ownMachine = (machines, intent) => {
|
|
1917
|
+
// Once an exact ID is durable, a same-name Machine is potentially a
|
|
1918
|
+
// replacement and must never inherit that identity. machineId may be an
|
|
1919
|
+
// observation; only machineCreatedByAttemptId grants automatic authority.
|
|
1920
|
+
const recordedId = this.recordedMachineId(intent);
|
|
1921
|
+
if (recordedId)
|
|
1922
|
+
return machines.find((machine) => machine.id === recordedId);
|
|
1923
|
+
if (options.recordedResourcesOnly)
|
|
1924
|
+
return undefined;
|
|
1925
|
+
const direct = machines.find((machine) => machine.name === intent.attemptId ||
|
|
1926
|
+
machine.config?.metadata?.[`${this.metadataPrefix}.createAttempt`] === intent.attemptId);
|
|
1927
|
+
if (direct || !intent.attemptId.startsWith('legacy:') || !intent.sandboxId)
|
|
1928
|
+
return direct;
|
|
1929
|
+
const prefix = `${intent.appName}:`;
|
|
1930
|
+
if (!intent.sandboxId.startsWith(prefix))
|
|
1931
|
+
return undefined;
|
|
1932
|
+
const sandboxMachineId = intent.sandboxId.slice(prefix.length);
|
|
1933
|
+
return machines.find((machine) => machine.id === sandboxMachineId);
|
|
1934
|
+
};
|
|
1935
|
+
const adoptLegacyResources = async (machine) => {
|
|
1936
|
+
if (!intent.attemptId.startsWith('legacy:'))
|
|
1937
|
+
return;
|
|
1938
|
+
const metadata = machine.config?.metadata ?? {};
|
|
1939
|
+
const logicalName = metadata[`${this.metadataPrefix}.volumeName`];
|
|
1940
|
+
for (const mount of machine.config?.mounts ?? []) {
|
|
1941
|
+
if (!mount.volume || intent.volumes.some((volume) => volume.id === mount.volume))
|
|
1942
|
+
continue;
|
|
1943
|
+
const listed = live.volumes.find((volume) => volume.id === mount.volume);
|
|
1944
|
+
await journal.recordVolume(intent.attemptId, {
|
|
1945
|
+
id: mount.volume,
|
|
1946
|
+
name: listed?.name ?? mount.volume,
|
|
1947
|
+
...(logicalName ? { logicalName } : {}),
|
|
1948
|
+
workspace: false,
|
|
1949
|
+
resolved: true,
|
|
1950
|
+
adoptedByAttemptId: intent.attemptId,
|
|
1951
|
+
});
|
|
1952
|
+
}
|
|
1953
|
+
const routes = decodePrivateRoutes(metadata[`${this.metadataPrefix}.${PRIVATE_ROUTES_METADATA_SUFFIX}`]);
|
|
1954
|
+
const network = this.networkFor(intent.appName);
|
|
1955
|
+
for (const [targetApp, ip] of Object.entries(routes)) {
|
|
1956
|
+
// A durable exact binding for the intended network is authoritative.
|
|
1957
|
+
// Otherwise validate the IP independently; copied metadata is only a hint.
|
|
1958
|
+
if (intent.addresses.some((address) => address.targetApp === targetApp &&
|
|
1959
|
+
address.network === network &&
|
|
1960
|
+
address.ip === ip &&
|
|
1961
|
+
address.resolved)) {
|
|
1962
|
+
continue;
|
|
1963
|
+
}
|
|
1964
|
+
const validated = await this.privateRouteMatchesNetwork(targetApp, ip, network, signal);
|
|
1965
|
+
await journal.recordAddress(intent.attemptId, {
|
|
1966
|
+
targetApp,
|
|
1967
|
+
network: network ?? intent.appName,
|
|
1968
|
+
ip,
|
|
1969
|
+
owned: validated,
|
|
1970
|
+
resolved: validated,
|
|
1971
|
+
});
|
|
1972
|
+
}
|
|
1973
|
+
};
|
|
1974
|
+
const assertObservedMachineUnreferenced = async (intent, machineId) => {
|
|
1975
|
+
if (intent.machineCreatedByAttemptId === intent.attemptId)
|
|
1976
|
+
return;
|
|
1977
|
+
const siblings = await journal.forApp(intent.appName);
|
|
1978
|
+
if (signal.aborted)
|
|
1979
|
+
throw signal.reason;
|
|
1980
|
+
const conflict = siblings.find((candidate) => candidate.attemptId !== intent.attemptId &&
|
|
1981
|
+
candidate.state !== 'abandoned' &&
|
|
1982
|
+
this.recordedMachineId(candidate) === machineId);
|
|
1983
|
+
if (conflict) {
|
|
1984
|
+
throw new Error(`Fly Machine "${machineId}" is still referenced by ${conflict.state} attempt "${conflict.attemptId}"; observed retirement is KEEP`);
|
|
1985
|
+
}
|
|
1986
|
+
};
|
|
1987
|
+
const activeStates = [
|
|
1988
|
+
'creating',
|
|
1989
|
+
'starting',
|
|
1990
|
+
'started',
|
|
1991
|
+
'restarting',
|
|
1992
|
+
'updating',
|
|
1993
|
+
'replacing',
|
|
1994
|
+
'migrated',
|
|
1995
|
+
];
|
|
1996
|
+
const hasActiveSibling = async (intent) => {
|
|
1997
|
+
if (signal.aborted)
|
|
1998
|
+
throw signal.reason;
|
|
1999
|
+
const siblings = await journal.forApp(intent.appName);
|
|
2000
|
+
if (signal.aborted)
|
|
2001
|
+
throw signal.reason;
|
|
2002
|
+
return siblings.some((candidate) => {
|
|
2003
|
+
if (candidate.attemptId === intent.attemptId)
|
|
2004
|
+
return false;
|
|
2005
|
+
if (candidate.state === 'serving')
|
|
2006
|
+
return true;
|
|
2007
|
+
// Every expired sibling in this set is independently reclaimable and
|
|
2008
|
+
// will be retired under its own lock in this pass. The current app lock
|
|
2009
|
+
// fences it now; treating the stale row as live deadlocks both rows.
|
|
2010
|
+
return (candidate.state === 'creating' && !options.expiredCreatingIds?.has(candidate.attemptId));
|
|
2011
|
+
});
|
|
2012
|
+
};
|
|
2013
|
+
const readRecordedMachine = async (current) => {
|
|
2014
|
+
const id = this.recordedMachineId(current);
|
|
2015
|
+
if (!id)
|
|
2016
|
+
return undefined;
|
|
2017
|
+
return this.requestForDestruction(`/apps/${current.appName}/machines/${id}`, { nullOn: [404] }, signal);
|
|
2018
|
+
};
|
|
2019
|
+
const releasedMachineIds = new Set();
|
|
2020
|
+
let intent = await fresh();
|
|
2021
|
+
let live = await this.readAttemptAppState(intent.appName, signal);
|
|
2022
|
+
const observed = ownMachine(options.observedMachines ?? [], intent);
|
|
2023
|
+
if (observed) {
|
|
2024
|
+
// destroyApp read this metadata before waiting for the attempt lock. Save
|
|
2025
|
+
// that observation now; a later list omission cannot erase ownership.
|
|
2026
|
+
intent = await fresh();
|
|
2027
|
+
await adoptLegacyResources(observed);
|
|
2028
|
+
intent = await fresh();
|
|
2029
|
+
}
|
|
2030
|
+
// A list omission cannot erase a durable ID. Read that exact resource;
|
|
2031
|
+
// 403/5xx keeps the row, 404 is authoritative, and a live result follows
|
|
2032
|
+
// the same state/provenance checks as a listed Machine.
|
|
2033
|
+
const machine = ownMachine(live.machines, intent) ??
|
|
2034
|
+
(!forget || mayDeleteApp ? undefined : ((await readRecordedMachine(intent)) ?? undefined));
|
|
2035
|
+
if (machine && !this.recordedMachineId(intent)) {
|
|
2036
|
+
// A name/metadata match recovers an exact identity as observation only.
|
|
2037
|
+
// It deliberately does not gain automatic deletion provenance.
|
|
2038
|
+
await journal.recordMachineObservation(intent.attemptId, generation, machine.id);
|
|
2039
|
+
intent = await fresh();
|
|
2040
|
+
}
|
|
2041
|
+
if (automatic &&
|
|
2042
|
+
machine &&
|
|
2043
|
+
(intent.machineCreatedByAttemptId !== intent.attemptId || intent.machineId !== machine.id)) {
|
|
2044
|
+
throw new Error(`Fly Machine "${machine.id}" for attempt "${intent.attemptId}" has no matching accepted-POST provenance; KEEP`);
|
|
2045
|
+
}
|
|
2046
|
+
if (failedMachineSweep && machine) {
|
|
2047
|
+
throw new Error(`Fly Machine "${machine.id}" appeared during failed-create sweep; KEEP`);
|
|
2048
|
+
}
|
|
2049
|
+
if (!retire &&
|
|
2050
|
+
machine &&
|
|
2051
|
+
(activeStates.includes(machine.state ?? 'created') ||
|
|
2052
|
+
intent.volumes.some((volume) => volume.workspace))) {
|
|
2053
|
+
return false;
|
|
2054
|
+
}
|
|
2055
|
+
if (machine) {
|
|
2056
|
+
// Re-read immediately before the destructive decision, not from the lock-wait snapshot.
|
|
2057
|
+
intent = await fresh();
|
|
2058
|
+
live = await this.readAttemptAppState(intent.appName, signal);
|
|
2059
|
+
const current = ownMachine(live.machines, intent) ??
|
|
2060
|
+
(!forget || mayDeleteApp ? undefined : ((await readRecordedMachine(intent)) ?? undefined));
|
|
2061
|
+
if (failedMachineSweep && current) {
|
|
2062
|
+
throw new Error(`Fly Machine "${current.id}" appeared during failed-create sweep; KEEP`);
|
|
2063
|
+
}
|
|
2064
|
+
if (!retire &&
|
|
2065
|
+
current &&
|
|
2066
|
+
(activeStates.includes(current.state ?? 'created') ||
|
|
2067
|
+
intent.volumes.some((volume) => volume.workspace))) {
|
|
2068
|
+
return false;
|
|
2069
|
+
}
|
|
2070
|
+
if (automatic &&
|
|
2071
|
+
current &&
|
|
2072
|
+
(intent.machineCreatedByAttemptId !== intent.attemptId || intent.machineId !== current.id)) {
|
|
2073
|
+
throw new Error(`Fly Machine "${current.id}" for attempt "${intent.attemptId}" lost accepted-POST provenance before deletion; KEEP`);
|
|
2074
|
+
}
|
|
2075
|
+
if (current) {
|
|
2076
|
+
// Migration 1793275400000 recorded only the synthetic attempt and
|
|
2077
|
+
// sandbox ID. The owned Machine's mounts/routes rebuild the exact-ID
|
|
2078
|
+
// responsibilities before its metadata disappears on deletion.
|
|
2079
|
+
await adoptLegacyResources(current);
|
|
2080
|
+
intent = await fresh();
|
|
2081
|
+
// Observation is not ownership. Recheck every other durable reference
|
|
2082
|
+
// on this same app lock immediately before the irreversible request.
|
|
2083
|
+
await assertObservedMachineUnreferenced(intent, current.id);
|
|
2084
|
+
await this.deleteRetired(intent, generation, `/apps/${intent.appName}/machines/${current.id}?force=true`, signal);
|
|
2085
|
+
releasedMachineIds.add(current.id);
|
|
2086
|
+
}
|
|
2087
|
+
}
|
|
2088
|
+
const releasedVolumeIds = new Set();
|
|
2089
|
+
for (const recorded of intent.volumes) {
|
|
2090
|
+
intent = await fresh();
|
|
2091
|
+
let volumeIntent = intent.volumes.find((candidate) => candidate.name === recorded.name || candidate.id === recorded.id);
|
|
2092
|
+
if (!volumeIntent || (volumeIntent.resolved === true && !volumeIntent.id))
|
|
2093
|
+
continue;
|
|
2094
|
+
live = await this.readAttemptAppState(intent.appName, signal);
|
|
2095
|
+
if (live.appGone)
|
|
2096
|
+
continue;
|
|
2097
|
+
if (options.recordedResourcesOnly && !volumeIntent.id) {
|
|
2098
|
+
throw new Error(`Fly volume "${volumeIntent.name}" for attempt "${intent.attemptId}" has no exact durable ID; refusing discovery during explicit retirement`);
|
|
2099
|
+
}
|
|
2100
|
+
if (!volumeIntent.id) {
|
|
2101
|
+
const matches = live.volumes.filter((candidate) => candidate.name === volumeIntent?.name);
|
|
2102
|
+
if (matches.length > 1) {
|
|
2103
|
+
throw new Error(`Fly volume create "${volumeIntent.name}" has ambiguous matches; an explicit ID is required`);
|
|
2104
|
+
}
|
|
2105
|
+
const discovered = matches[0];
|
|
2106
|
+
if (!discovered) {
|
|
2107
|
+
throw new Error(`Fly volume create "${volumeIntent.name}" remains unresolved`);
|
|
2108
|
+
}
|
|
2109
|
+
// A unique name match is an observation only. Persist its exact ID for
|
|
2110
|
+
// deliberate retirement, never accepted-POST/automatic-delete authority.
|
|
2111
|
+
await journal.recordVolume(intent.attemptId, {
|
|
2112
|
+
...volumeIntent,
|
|
2113
|
+
id: discovered.id,
|
|
2114
|
+
resolved: true,
|
|
2115
|
+
createdByAttemptId: undefined,
|
|
2116
|
+
adoptedByAttemptId: intent.attemptId,
|
|
2117
|
+
});
|
|
2118
|
+
intent = await fresh();
|
|
2119
|
+
volumeIntent = intent.volumes.find((candidate) => candidate.id === discovered.id);
|
|
2120
|
+
}
|
|
2121
|
+
if (automatic &&
|
|
2122
|
+
volumeIntent &&
|
|
2123
|
+
(volumeIntent.createdByAttemptId !== intent.attemptId || volumeIntent.resolved !== true)) {
|
|
2124
|
+
throw new Error(`Fly volume "${volumeIntent.id ?? volumeIntent.name}" for attempt "${intent.attemptId}" has no matching accepted-POST provenance; KEEP`);
|
|
2125
|
+
}
|
|
2126
|
+
if (!volumeIntent?.id || (!retire && volumeIntent.workspace))
|
|
2127
|
+
continue;
|
|
2128
|
+
// ID, attachment, workspace flag and ownership are all re-read under the lock.
|
|
2129
|
+
intent = await fresh();
|
|
2130
|
+
volumeIntent = intent.volumes.find((candidate) => candidate.id === volumeIntent?.id);
|
|
2131
|
+
live = await this.readAttemptAppState(intent.appName, signal);
|
|
2132
|
+
const volume = live.volumes.find((candidate) => candidate.id === volumeIntent?.id);
|
|
2133
|
+
if (failedMachineSweep && volumeIntent?.id) {
|
|
2134
|
+
const referencing = live.machines.find((machine) => machine.config?.mounts?.some((mount) => mount.volume === volumeIntent?.id));
|
|
2135
|
+
if (referencing || volume?.attached_machine_id) {
|
|
2136
|
+
throw new Error(`Fly volume "${volumeIntent.id}" gained a Machine reference during sweep; KEEP`);
|
|
2137
|
+
}
|
|
2138
|
+
}
|
|
2139
|
+
if (automatic &&
|
|
2140
|
+
volumeIntent &&
|
|
2141
|
+
(volumeIntent.createdByAttemptId !== intent.attemptId || volumeIntent.resolved !== true)) {
|
|
2142
|
+
throw new Error(`Fly volume "${volumeIntent.id ?? volumeIntent.name}" for attempt "${intent.attemptId}" lost accepted-POST provenance before deletion; KEEP`);
|
|
2143
|
+
}
|
|
2144
|
+
if (!volumeIntent?.id || (!retire && volumeIntent.workspace) || volume?.attached_machine_id) {
|
|
2145
|
+
continue;
|
|
2146
|
+
}
|
|
2147
|
+
// A list omission is not deletion. The durable exact ID is sufficient
|
|
2148
|
+
// capability for an idempotent DELETE; success/404, never the list,
|
|
2149
|
+
// authorizes forgetting it. Fly still rejects a really attached volume.
|
|
2150
|
+
const volumeId = volumeIntent.id;
|
|
2151
|
+
await this.deleteRetired(intent, generation, `/apps/${intent.appName}/volumes/${volumeId}`, signal);
|
|
2152
|
+
releasedVolumeIds.add(volumeId);
|
|
2153
|
+
}
|
|
2154
|
+
intent = await fresh();
|
|
2155
|
+
live = await this.readAttemptAppState(intent.appName, signal);
|
|
2156
|
+
if (intent.addresses.length &&
|
|
2157
|
+
(live.machines.some((candidate) => candidate.id !== machine?.id) ||
|
|
2158
|
+
(await hasActiveSibling(intent)))) {
|
|
2159
|
+
return false;
|
|
2160
|
+
}
|
|
2161
|
+
for (const recorded of intent.addresses) {
|
|
2162
|
+
intent = await fresh();
|
|
2163
|
+
let address = intent.addresses.find((candidate) => candidate.targetApp === recorded.targetApp && candidate.network === recorded.network);
|
|
2164
|
+
if (!address || (address.resolved && !address.owned))
|
|
2165
|
+
continue;
|
|
2166
|
+
if (options.recordedResourcesOnly && (!address.owned || !address.ip)) {
|
|
2167
|
+
throw new Error(`Fly address for attempt "${intent.attemptId}" on "${address.targetApp}" has no exact owned IP; refusing discovery during explicit retirement`);
|
|
2168
|
+
}
|
|
2169
|
+
if (!address.resolved) {
|
|
2170
|
+
// Only an unresolved create needs discovery. A known owned IP is its
|
|
2171
|
+
// own DELETE capability, even when the target app no longer exists.
|
|
2172
|
+
const assignments = await this.listPrivateAssignments(address.targetApp, signal);
|
|
2173
|
+
if (!assignments) {
|
|
2174
|
+
throw new Error(`Could not resolve Fly addresses on "${address.targetApp}"`);
|
|
2175
|
+
}
|
|
2176
|
+
const assignment = assignments.find((candidate) => candidate.network === address?.network);
|
|
2177
|
+
if (!assignment?.ip) {
|
|
2178
|
+
throw new Error(`Fly address allocation for attempt "${intent.attemptId}" is unresolved`);
|
|
2179
|
+
}
|
|
2180
|
+
await journal.recordAddress(intent.attemptId, {
|
|
2181
|
+
...address,
|
|
2182
|
+
ip: assignment.ip,
|
|
2183
|
+
owned: true,
|
|
2184
|
+
resolved: true,
|
|
2185
|
+
});
|
|
2186
|
+
intent = await fresh();
|
|
2187
|
+
address = intent.addresses.find((candidate) => candidate.targetApp === recorded.targetApp && candidate.network === recorded.network);
|
|
2188
|
+
}
|
|
2189
|
+
if (!address?.owned)
|
|
2190
|
+
continue;
|
|
2191
|
+
if (!address.ip)
|
|
2192
|
+
throw new Error(`Owned Fly address for "${intent.attemptId}" has no IP`);
|
|
2193
|
+
// The durable exact IP, not list presence, is the release capability.
|
|
2194
|
+
// A retry after a lost successful DELETE gets 404 and completes below.
|
|
2195
|
+
intent = await fresh();
|
|
2196
|
+
address = intent.addresses.find((candidate) => candidate.targetApp === recorded.targetApp && candidate.network === recorded.network);
|
|
2197
|
+
if (!address?.owned)
|
|
2198
|
+
continue;
|
|
2199
|
+
if (!address.ip)
|
|
2200
|
+
throw new Error(`Owned Fly address for "${intent.attemptId}" has no IP`);
|
|
2201
|
+
await this.deleteRetired(intent, generation, `/apps/${address.targetApp}/ip_assignments/${encodeURIComponent(address.ip)}`, signal);
|
|
2202
|
+
await journal.recordAddress(intent.attemptId, {
|
|
2203
|
+
...address,
|
|
2204
|
+
owned: false,
|
|
2205
|
+
resolved: true,
|
|
2206
|
+
});
|
|
2207
|
+
}
|
|
2208
|
+
intent = await fresh();
|
|
2209
|
+
live = await this.readAttemptAppState(intent.appName, signal);
|
|
2210
|
+
const ownedVolumesRemain = intent.volumes.some((volume) => {
|
|
2211
|
+
if (volume.resolved === true && !volume.id)
|
|
2212
|
+
return false;
|
|
2213
|
+
if (live.appGone)
|
|
2214
|
+
return false;
|
|
2215
|
+
if (volume.id)
|
|
2216
|
+
return !releasedVolumeIds.has(volume.id);
|
|
2217
|
+
return live.volumes.some((candidate) => candidate.name === volume.name);
|
|
2218
|
+
});
|
|
2219
|
+
// Exact app 404 proves absence of every contained resource, including an
|
|
2220
|
+
// unresolved volume. External addresses still settle independently below.
|
|
2221
|
+
const unsettledVolume = !live.appGone && intent.volumes.some((volume) => volume.resolved !== true);
|
|
2222
|
+
if (failedMachineSweep && this.appPerProject() && live.machines.length) {
|
|
2223
|
+
throw new Error(`Fly app "${intent.appName}" gained a Machine during sweep; KEEP`);
|
|
2224
|
+
}
|
|
2225
|
+
if (failedMachineSweep && deleteApp && this.appPerProject() && live.volumes.length) {
|
|
2226
|
+
throw new Error(`Fly app "${intent.appName}" is not empty; failed-create sweep is KEEP`);
|
|
2227
|
+
}
|
|
2228
|
+
const listedMachine = ownMachine(live.machines, intent);
|
|
2229
|
+
if (unsettledVolume ||
|
|
2230
|
+
ownedVolumesRemain ||
|
|
2231
|
+
(listedMachine && !releasedMachineIds.has(listedMachine.id))) {
|
|
2232
|
+
return false;
|
|
2233
|
+
}
|
|
2234
|
+
if (intent.addresses.some((address) => !address.resolved || address.owned))
|
|
2235
|
+
return false;
|
|
2236
|
+
if (mayDeleteApp && !live.appGone && (await hasActiveSibling(intent))) {
|
|
2237
|
+
return false;
|
|
2238
|
+
}
|
|
2239
|
+
let appSettled = live.appGone;
|
|
2240
|
+
if (mayDeleteApp && !live.appGone) {
|
|
2241
|
+
// Deliberate retirement alone may delete a container. Addresses above are
|
|
2242
|
+
// settled first; provenance and every Machine are re-read under the lock.
|
|
2243
|
+
intent = await fresh();
|
|
2244
|
+
live = await this.readAttemptAppState(intent.appName, signal);
|
|
2245
|
+
if (failedMachineSweep && (live.machines.length || live.volumes.length)) {
|
|
2246
|
+
throw new Error(`Fly app "${intent.appName}" is not empty before app deletion; KEEP`);
|
|
2247
|
+
}
|
|
2248
|
+
if (automatic &&
|
|
2249
|
+
(intent.appCreatedByAttemptId !== intent.attemptId || intent.appCreateResolved !== true)) {
|
|
2250
|
+
throw new Error(`Fly app "${intent.appName}" for attempt "${intent.attemptId}" has no matching accepted-POST provenance; KEEP`);
|
|
2251
|
+
}
|
|
2252
|
+
if (live.machines.length || ((automatic || !retire) && live.volumes.length))
|
|
2253
|
+
return false;
|
|
2254
|
+
await this.deleteRetired(intent, generation, `/apps/${intent.appName}`, signal);
|
|
2255
|
+
appSettled = true;
|
|
2256
|
+
}
|
|
2257
|
+
if (automatic && appRequired && !appSettled && !this.hasUnresolvedMachineIdentity(intent)) {
|
|
2258
|
+
return false;
|
|
2259
|
+
}
|
|
2260
|
+
if (!forget)
|
|
2261
|
+
return false;
|
|
2262
|
+
intent = await fresh();
|
|
2263
|
+
await this.assertAuthoritativeForget(intent, signal, { machineIds: releasedMachineIds, volumeIds: releasedVolumeIds }, { required: appRequired, settled: appSettled });
|
|
2264
|
+
await journal.forget(intent.attemptId, generation);
|
|
2265
|
+
return true;
|
|
2266
|
+
}
|
|
2267
|
+
/** Lock and settle each eligible attempt independently; there is no lock-free mode. */
|
|
2268
|
+
async reconcileCreateAttempts(options = {}) {
|
|
2269
|
+
const journal = this.config.createAttemptJournal;
|
|
2270
|
+
if (!journal)
|
|
2271
|
+
return { deleted: 0, failed: 0 };
|
|
2272
|
+
const now = options.now ?? Date.now();
|
|
2273
|
+
const timeoutMs = options.createTimeoutMs ?? 15 * 60_000;
|
|
2274
|
+
const cutoff = new Date(now - timeoutMs);
|
|
2275
|
+
const intents = (await journal.reclaimable(cutoff)).filter((intent) => !options.attemptId || intent.attemptId === options.attemptId);
|
|
2276
|
+
const expiredCreatingIds = new Set(intents.filter((intent) => intent.state === 'creating').map((intent) => intent.attemptId));
|
|
2277
|
+
const summary = { deleted: 0, failed: 0 };
|
|
2278
|
+
for (const intent of intents) {
|
|
2279
|
+
try {
|
|
2280
|
+
if (!journal.withLock)
|
|
2281
|
+
throw new Error('Attempt settlement requires its lifecycle lock');
|
|
2282
|
+
const forgotten = await journal.withLock(intent, async (signal, generation) => {
|
|
2283
|
+
// Selection happened before the lock wait. Re-read state and expiry;
|
|
2284
|
+
// only failed/expired attempts may be retired by this automatic pass.
|
|
2285
|
+
const current = await journal.get(intent.attemptId);
|
|
2286
|
+
if (!current)
|
|
2287
|
+
return false;
|
|
2288
|
+
if (current.state === 'serving')
|
|
2289
|
+
return false;
|
|
2290
|
+
if (current.state === 'creating') {
|
|
2291
|
+
const expired = (await journal.reclaimable(cutoff)).some((candidate) => candidate.attemptId === intent.attemptId);
|
|
2292
|
+
if (!expired)
|
|
2293
|
+
return false;
|
|
2294
|
+
}
|
|
2295
|
+
if (signal.aborted)
|
|
2296
|
+
throw signal.reason;
|
|
2297
|
+
const failedMachineSweep = current.machineCreateFailed === true;
|
|
2298
|
+
if (failedMachineSweep)
|
|
2299
|
+
await this.assertFailedMachineSweepable(current, signal);
|
|
2300
|
+
if (current.state === 'failed' || current.state === 'creating') {
|
|
2301
|
+
await journal.retire(current.attemptId, generation);
|
|
2302
|
+
}
|
|
2303
|
+
if (signal.aborted)
|
|
2304
|
+
throw signal.reason;
|
|
2305
|
+
const settled = await this.settleCreateAttempt(current, signal, generation, {
|
|
2306
|
+
automatic: true,
|
|
2307
|
+
retire: failedMachineSweep,
|
|
2308
|
+
failedMachineSweep,
|
|
2309
|
+
expiredCreatingIds,
|
|
2310
|
+
});
|
|
2311
|
+
if (failedMachineSweep && !settled) {
|
|
2312
|
+
throw new Error(`Fly failed-create sweep for "${current.attemptId}" remains incomplete`);
|
|
2313
|
+
}
|
|
2314
|
+
return settled;
|
|
2315
|
+
});
|
|
2316
|
+
if (forgotten)
|
|
2317
|
+
summary.deleted++;
|
|
2318
|
+
}
|
|
2319
|
+
catch (error) {
|
|
2320
|
+
// A state transition back to creating/serving is a safe skip, not a cleanup failure.
|
|
2321
|
+
if (!/no longer settleable/.test(error instanceof Error ? error.message : '')) {
|
|
2322
|
+
summary.failed++;
|
|
2323
|
+
logger.warn('Could not reconcile a Fly create attempt; retaining its durable intent', {
|
|
2324
|
+
attemptId: intent.attemptId,
|
|
2325
|
+
app: intent.appName,
|
|
2326
|
+
error,
|
|
2327
|
+
});
|
|
2328
|
+
}
|
|
2329
|
+
}
|
|
2330
|
+
}
|
|
2331
|
+
return summary;
|
|
2332
|
+
}
|
|
2333
|
+
/** Re-read and abandon only while the attempt's current lifecycle lock is live. */
|
|
2334
|
+
async retireCreateAttempt(snapshot, outerSignal, options) {
|
|
2335
|
+
const journal = this.config.createAttemptJournal;
|
|
2336
|
+
if (!journal?.withLock)
|
|
2337
|
+
throw new Error('Attempt settlement requires its lifecycle lock');
|
|
2338
|
+
return journal.withLock(snapshot, async (lockSignal, generation) => {
|
|
2339
|
+
const signal = AbortSignal.any([outerSignal, lockSignal]);
|
|
2340
|
+
if (signal.aborted)
|
|
2341
|
+
throw signal.reason;
|
|
2342
|
+
const current = await journal.get(snapshot.attemptId);
|
|
2343
|
+
if (signal.aborted)
|
|
2344
|
+
throw signal.reason;
|
|
2345
|
+
if (!current)
|
|
2346
|
+
return false;
|
|
2347
|
+
if ((options.expectedAppName && current.appName !== options.expectedAppName) ||
|
|
2348
|
+
(options.expectedMachineId && this.recordedMachineId(current) !== options.expectedMachineId)) {
|
|
2349
|
+
throw new Error(`Fly create attempt "${snapshot.attemptId}" changed outside the caller-validated scope`);
|
|
2350
|
+
}
|
|
2351
|
+
if (current.state !== 'abandoned') {
|
|
2352
|
+
if (signal.aborted)
|
|
2353
|
+
throw signal.reason;
|
|
2354
|
+
await journal.retire(current.attemptId, generation);
|
|
2355
|
+
if (signal.aborted)
|
|
2356
|
+
throw signal.reason;
|
|
2357
|
+
}
|
|
2358
|
+
return this.settleCreateAttempt(current, signal, generation, {
|
|
2359
|
+
retire: true,
|
|
2360
|
+
...options,
|
|
2361
|
+
});
|
|
2362
|
+
});
|
|
2363
|
+
}
|
|
2364
|
+
/** Journal a pre-journal Machine before its first destructive request. */
|
|
2365
|
+
async journalLegacyMachine(app, machine, fallbackProjectId) {
|
|
2366
|
+
const journal = this.config.createAttemptJournal;
|
|
2367
|
+
if (!journal)
|
|
2368
|
+
throw new Error('Legacy Machine adoption requires its durable journal');
|
|
2369
|
+
const metadata = machine.config?.metadata ?? {};
|
|
2370
|
+
let projectId = metadata[`${this.metadataPrefix}.projectId`] ?? fallbackProjectId;
|
|
2371
|
+
if (!projectId && this.appPerProject()) {
|
|
2372
|
+
const prefix = `${this.appPrefix()}-`;
|
|
2373
|
+
const candidate = app.startsWith(prefix) ? app.slice(prefix.length) : '';
|
|
2374
|
+
if (candidate && appNameForProject(this.appPrefix(), candidate) === app) {
|
|
2375
|
+
projectId = candidate;
|
|
2376
|
+
}
|
|
2377
|
+
}
|
|
2378
|
+
if (!projectId) {
|
|
2379
|
+
throw new Error(`Refusing to destroy legacy Machine "${machine.id}" without its project identity`);
|
|
2380
|
+
}
|
|
2381
|
+
const sandboxId = `${app}:${machine.id}`;
|
|
2382
|
+
const attemptId = `legacy:${sandboxId}`;
|
|
2383
|
+
const existing = await journal.get(attemptId);
|
|
2384
|
+
if (existing) {
|
|
2385
|
+
await journal.markMachineAdopted(attemptId);
|
|
2386
|
+
if (existing.sandboxId) {
|
|
2387
|
+
const adopted = await journal.get(attemptId);
|
|
2388
|
+
if (!adopted)
|
|
2389
|
+
throw new Error(`Could not persist legacy ownership for "${sandboxId}"`);
|
|
2390
|
+
return adopted;
|
|
2391
|
+
}
|
|
2392
|
+
await journal.markState(attemptId, existing.state === 'creating' ? 'serving' : existing.state, sandboxId);
|
|
2393
|
+
const repaired = await journal.get(attemptId);
|
|
2394
|
+
if (!repaired?.sandboxId) {
|
|
2395
|
+
throw new Error(`Could not repair legacy ownership for "${sandboxId}"`);
|
|
2396
|
+
}
|
|
2397
|
+
return repaired;
|
|
2398
|
+
}
|
|
2399
|
+
await journal.begin({
|
|
2400
|
+
attemptId,
|
|
2401
|
+
projectId,
|
|
2402
|
+
ownerProjectId: metadata['molecule.production'] ?? projectId,
|
|
2403
|
+
appName: app,
|
|
2404
|
+
state: 'serving',
|
|
2405
|
+
sandboxId,
|
|
2406
|
+
machineAdopted: true,
|
|
2407
|
+
});
|
|
2408
|
+
const adopted = await journal.get(attemptId);
|
|
2409
|
+
if (!adopted)
|
|
2410
|
+
throw new Error(`Could not persist legacy ownership for "${sandboxId}"`);
|
|
2411
|
+
return adopted;
|
|
2412
|
+
}
|
|
2413
|
+
/** Journal an exclusively owned empty app rather than treating absence as DELETE authority. */
|
|
2414
|
+
async journalLegacyApp(app, projectId) {
|
|
2415
|
+
const journal = this.config.createAttemptJournal;
|
|
2416
|
+
if (!journal)
|
|
2417
|
+
throw new Error('Legacy app adoption requires its durable journal');
|
|
2418
|
+
const attemptId = `legacy:${app}:app`;
|
|
2419
|
+
const existing = await journal.get(attemptId);
|
|
2420
|
+
if (existing)
|
|
2421
|
+
return existing;
|
|
2422
|
+
await journal.begin({ attemptId, projectId, appName: app });
|
|
2423
|
+
await journal.markState(attemptId, 'serving');
|
|
2424
|
+
const adopted = await journal.get(attemptId);
|
|
2425
|
+
if (!adopted)
|
|
2426
|
+
throw new Error(`Could not persist legacy ownership for app "${app}"`);
|
|
2427
|
+
return adopted;
|
|
2428
|
+
}
|
|
2429
|
+
/**
|
|
2430
|
+
* Explicitly destroy an exclusively owned app and its workspace storage.
|
|
2431
|
+
* Recorded external addresses are released before any intent may be forgotten.
|
|
2432
|
+
*/
|
|
2433
|
+
async destroyApp(projectId, expectedAppName, signal) {
|
|
2434
|
+
if (!signal)
|
|
2435
|
+
throw new Error('Fly app destruction requires its lifecycle lock signal');
|
|
2436
|
+
if (!this.appPerProject()) {
|
|
2437
|
+
throw new Error('Refusing to delete a shared Fly app as though a caller exclusively owned it');
|
|
2438
|
+
}
|
|
2439
|
+
const app = appNameForProject(this.appPrefix(), projectId);
|
|
2440
|
+
if (expectedAppName !== undefined && expectedAppName !== app) {
|
|
2441
|
+
throw new Error(`Refusing to delete Fly app "${expectedAppName}": its durable identity resolves to "${app}"`);
|
|
2442
|
+
}
|
|
2443
|
+
const listed = await this.requestForDestruction(`/apps/${app}/machines`, { nullOn: [404] }, signal);
|
|
2444
|
+
const machines = Array.isArray(listed) ? listed : [];
|
|
2445
|
+
const routes = {};
|
|
2446
|
+
const attemptIds = new Set();
|
|
2447
|
+
const metadataHints = [];
|
|
2448
|
+
for (const machine of machines) {
|
|
2449
|
+
Object.assign(routes, decodePrivateRoutes(machine.config?.metadata?.[`${this.metadataPrefix}.${PRIVATE_ROUTES_METADATA_SUFFIX}`]));
|
|
2450
|
+
const attemptId = machine.config?.metadata?.[`${this.metadataPrefix}.createAttempt`];
|
|
2451
|
+
if (attemptId) {
|
|
2452
|
+
attemptIds.add(attemptId);
|
|
2453
|
+
metadataHints.push({ attemptId, machineId: machine.id });
|
|
2454
|
+
}
|
|
2455
|
+
}
|
|
2456
|
+
const journal = this.config.createAttemptJournal;
|
|
2457
|
+
if (journal) {
|
|
2458
|
+
const durable = new Map();
|
|
2459
|
+
const expectedMachineIds = new Map();
|
|
2460
|
+
const journaled = await journal.forApp(app);
|
|
2461
|
+
if (signal.aborted)
|
|
2462
|
+
throw signal.reason;
|
|
2463
|
+
for (const intent of journaled)
|
|
2464
|
+
durable.set(intent.attemptId, intent);
|
|
2465
|
+
for (const hint of metadataHints) {
|
|
2466
|
+
const intent = await journal.get(hint.attemptId);
|
|
2467
|
+
if (signal.aborted)
|
|
2468
|
+
throw signal.reason;
|
|
2469
|
+
// Metadata is only a hint. Both the requested app and this listed
|
|
2470
|
+
// Machine's exact durable identity must match before selection.
|
|
2471
|
+
if (intent?.appName === app && this.recordedMachineId(intent) === hint.machineId) {
|
|
2472
|
+
durable.set(intent.attemptId, intent);
|
|
2473
|
+
expectedMachineIds.set(intent.attemptId, hint.machineId);
|
|
2474
|
+
}
|
|
2475
|
+
else if (intent) {
|
|
2476
|
+
throw new Error(`Refusing copied createAttempt metadata for Machine "${hint.machineId}" in app "${app}"`);
|
|
2477
|
+
}
|
|
2478
|
+
}
|
|
2479
|
+
for (const machine of machines) {
|
|
2480
|
+
const createAttemptId = machine.config?.metadata?.[`${this.metadataPrefix}.createAttempt`];
|
|
2481
|
+
const known = [...durable.values()].some((intent) => intent.appName === app &&
|
|
2482
|
+
((intent.attemptId === createAttemptId &&
|
|
2483
|
+
this.recordedMachineId(intent) === machine.id) ||
|
|
2484
|
+
intent.sandboxId === `${app}:${machine.id}`));
|
|
2485
|
+
if (known)
|
|
2486
|
+
continue;
|
|
2487
|
+
const adopted = await this.journalLegacyMachine(app, machine, projectId);
|
|
2488
|
+
durable.set(adopted.attemptId, adopted);
|
|
2489
|
+
}
|
|
2490
|
+
if (!durable.size) {
|
|
2491
|
+
const adopted = await this.journalLegacyApp(app, projectId);
|
|
2492
|
+
durable.set(adopted.attemptId, adopted);
|
|
2493
|
+
}
|
|
2494
|
+
if (durable.size && !journal.withLock) {
|
|
2495
|
+
throw new Error(`Refusing to retire Fly app "${app}" without its lifecycle lock`);
|
|
2496
|
+
}
|
|
2497
|
+
if (durable.size && journal.withLock) {
|
|
2498
|
+
// Pass one settles every durable address and retires Machines without
|
|
2499
|
+
// deleting the app. No app deletion can bypass another attempt's failed
|
|
2500
|
+
// address release. Pass two deletes the app and forgets settled intents.
|
|
2501
|
+
for (const deleteApp of [false, true]) {
|
|
2502
|
+
for (const snapshot of durable.values()) {
|
|
2503
|
+
const current = await journal.get(snapshot.attemptId);
|
|
2504
|
+
if (signal.aborted)
|
|
2505
|
+
throw signal.reason;
|
|
2506
|
+
if (!current)
|
|
2507
|
+
continue;
|
|
2508
|
+
await this.retireCreateAttempt(current, signal, {
|
|
2509
|
+
deleteApp,
|
|
2510
|
+
forget: deleteApp,
|
|
2511
|
+
observedMachines: machines,
|
|
2512
|
+
expectedAppName: app,
|
|
2513
|
+
expectedMachineId: expectedMachineIds.get(current.attemptId),
|
|
2514
|
+
});
|
|
2515
|
+
}
|
|
2516
|
+
}
|
|
2517
|
+
const retained = (await Promise.all([...durable.keys()].map(async (attemptId) => journal.get(attemptId)))).some(Boolean);
|
|
2518
|
+
if (signal.aborted)
|
|
2519
|
+
throw signal.reason;
|
|
2520
|
+
if (retained)
|
|
2521
|
+
throw new Error(`Fly app "${app}" retirement has not fully settled`);
|
|
2522
|
+
return;
|
|
2523
|
+
}
|
|
2524
|
+
}
|
|
2525
|
+
// Legacy unjournaled apps retain explicit destruction. Never force an app
|
|
2526
|
+
// through a Machine that has no committed retired row: explicitly delete
|
|
2527
|
+
// each observed Machine, release its routes, then re-list before app DELETE.
|
|
2528
|
+
for (const machine of machines) {
|
|
2529
|
+
await this.requestForDestruction(`/apps/${app}/machines/${machine.id}?force=true`, { method: 'DELETE', nullOn: [404] }, signal);
|
|
2530
|
+
}
|
|
2531
|
+
for (const [targetApp, ip] of Object.entries(routes)) {
|
|
2532
|
+
if (!(await this.privateRouteMatchesNetwork(targetApp, ip, this.networkFor(app), signal))) {
|
|
2533
|
+
logger.warn('Kept a Fly private (Flycast) address with unvalidated network ownership', {
|
|
2534
|
+
targetApp,
|
|
2535
|
+
ip,
|
|
2536
|
+
app,
|
|
2537
|
+
});
|
|
2538
|
+
continue;
|
|
2539
|
+
}
|
|
2540
|
+
await this.requestForDestruction(`/apps/${targetApp}/ip_assignments/${encodeURIComponent(ip)}`, { method: 'DELETE', nullOn: [404] }, signal);
|
|
2541
|
+
}
|
|
2542
|
+
const remaining = await this.readAttemptAppState(app, signal);
|
|
2543
|
+
if (remaining.machines.length) {
|
|
2544
|
+
throw new Error(`Refusing to delete Fly app "${app}" while a Machine remains`);
|
|
2545
|
+
}
|
|
2546
|
+
if (!remaining.appGone) {
|
|
2547
|
+
await this.requestForDestruction(`/apps/${app}`, { method: 'DELETE', nullOn: [404] }, signal);
|
|
2548
|
+
}
|
|
2549
|
+
for (const attemptId of attemptIds) {
|
|
2550
|
+
if (signal.aborted)
|
|
2551
|
+
throw signal.reason;
|
|
2552
|
+
await journal?.forget(attemptId);
|
|
2553
|
+
}
|
|
2554
|
+
}
|
|
2555
|
+
/**
|
|
2556
|
+
* Deliberately retire one named shared-app attempt using only exact durable
|
|
2557
|
+
* resource IDs. Unlike legacy destroy(), this never infers ownership from a
|
|
2558
|
+
* Machine name, metadata, sandboxId, a volume name, or an address listing.
|
|
2559
|
+
*/
|
|
2560
|
+
async destroyAttempt(attemptId, signal) {
|
|
2561
|
+
if (!signal)
|
|
2562
|
+
throw new Error('Fly attempt destruction requires its lifecycle lock signal');
|
|
2563
|
+
if (this.appPerProject()) {
|
|
2564
|
+
throw new Error('Use explicit per-project app destruction for a per-project Fly attempt');
|
|
2565
|
+
}
|
|
2566
|
+
const journal = this.config.createAttemptJournal;
|
|
2567
|
+
if (!journal)
|
|
2568
|
+
throw new Error('Fly attempt destruction requires its durable journal');
|
|
2569
|
+
const snapshot = await journal.get(attemptId);
|
|
2570
|
+
if (signal.aborted)
|
|
2571
|
+
throw signal.reason;
|
|
2572
|
+
if (!snapshot)
|
|
2573
|
+
return;
|
|
2574
|
+
const forgotten = await this.retireCreateAttempt(snapshot, signal, {
|
|
2575
|
+
deleteApp: false,
|
|
2576
|
+
forget: true,
|
|
2577
|
+
recordedResourcesOnly: true,
|
|
2578
|
+
});
|
|
2579
|
+
if (!forgotten) {
|
|
2580
|
+
throw new Error(`Could not complete exact-resource cleanup for attempt "${attemptId}"; its intent was retained`);
|
|
2581
|
+
}
|
|
2582
|
+
}
|
|
1500
2583
|
/**
|
|
1501
2584
|
* Destroys a sandbox.
|
|
1502
2585
|
*
|
|
@@ -1514,15 +2597,19 @@ class FlyioSandboxProvider {
|
|
|
1514
2597
|
* other than a 404 (which counts as already gone), naming the failed step and
|
|
1515
2598
|
* HTTP status, so the caller can retry instead of forgetting a billed app.
|
|
1516
2599
|
*/
|
|
1517
|
-
async destroy(id) {
|
|
2600
|
+
async destroy(id, signal) {
|
|
2601
|
+
if (!signal)
|
|
2602
|
+
throw new Error('Fly destruction requires its lifecycle lock signal');
|
|
1518
2603
|
const { app, machineId } = parseSandboxId(id);
|
|
1519
2604
|
let volumeIds = [];
|
|
1520
2605
|
let privateRoutes = {};
|
|
2606
|
+
let createAttemptId;
|
|
2607
|
+
let observedMachine = null;
|
|
1521
2608
|
try {
|
|
1522
|
-
const machine = await this.
|
|
1523
|
-
|
|
1524
|
-
});
|
|
2609
|
+
const machine = await this.requestForDestruction(`/apps/${app}/machines/${machineId}`, { nullOn: [404] }, signal);
|
|
2610
|
+
observedMachine = machine;
|
|
1525
2611
|
privateRoutes = decodePrivateRoutes(machine?.config?.metadata?.[`${this.metadataPrefix}.${PRIVATE_ROUTES_METADATA_SUFFIX}`]);
|
|
2612
|
+
createAttemptId = machine?.config?.metadata?.[`${this.metadataPrefix}.createAttempt`];
|
|
1526
2613
|
if (!this.appPerProject()) {
|
|
1527
2614
|
volumeIds = (machine?.config?.mounts ?? [])
|
|
1528
2615
|
.map((mount) => mount.volume)
|
|
@@ -1530,19 +2617,60 @@ class FlyioSandboxProvider {
|
|
|
1530
2617
|
}
|
|
1531
2618
|
}
|
|
1532
2619
|
catch (error) {
|
|
2620
|
+
if (signal.aborted)
|
|
2621
|
+
throw signal.reason;
|
|
1533
2622
|
// Best-effort metadata gathering; the Machine may already be gone.
|
|
1534
2623
|
// Destruction proceeds either way — but say so, because an unread
|
|
1535
2624
|
// metadata block means the private addresses below are NOT released.
|
|
1536
2625
|
logger.warn('Could not read the Fly Machine before destroy — its volumes and any private (Flycast) ' +
|
|
1537
2626
|
'addresses it recorded will not be cleaned up', { id, error });
|
|
1538
2627
|
}
|
|
2628
|
+
const journal = this.config.createAttemptJournal;
|
|
2629
|
+
if (journal) {
|
|
2630
|
+
const durable = await journal.forApp(app);
|
|
2631
|
+
if (signal.aborted)
|
|
2632
|
+
throw signal.reason;
|
|
2633
|
+
// Metadata is only a lookup hint. It selects an attempt when its durable
|
|
2634
|
+
// exact Machine identity also names the caller's ID; copied metadata must
|
|
2635
|
+
// never redirect retirement to a different Machine's attempt.
|
|
2636
|
+
let intent = durable.find((candidate) => candidate.attemptId === createAttemptId &&
|
|
2637
|
+
this.recordedMachineId(candidate) === machineId) ?? durable.find((candidate) => candidate.sandboxId === id);
|
|
2638
|
+
if (!intent && observedMachine) {
|
|
2639
|
+
intent = await this.journalLegacyMachine(app, observedMachine);
|
|
2640
|
+
}
|
|
2641
|
+
if (intent) {
|
|
2642
|
+
const current = await journal.get(intent.attemptId);
|
|
2643
|
+
if (signal.aborted)
|
|
2644
|
+
throw signal.reason;
|
|
2645
|
+
if (!current)
|
|
2646
|
+
throw new Error(`Fly create attempt "${intent.attemptId}" disappeared`);
|
|
2647
|
+
const durableMachineId = this.recordedMachineId(current);
|
|
2648
|
+
if (durableMachineId !== machineId) {
|
|
2649
|
+
throw new Error(`Refusing to retire attempt "${current.attemptId}": caller named Machine "${machineId}" but durable identity is "${durableMachineId ?? 'unresolved'}"`);
|
|
2650
|
+
}
|
|
2651
|
+
const forgotten = await this.retireCreateAttempt(current, signal, {
|
|
2652
|
+
deleteApp: this.appPerProject(),
|
|
2653
|
+
forget: true,
|
|
2654
|
+
observedMachines: observedMachine ? [observedMachine] : [],
|
|
2655
|
+
expectedAppName: app,
|
|
2656
|
+
expectedMachineId: machineId,
|
|
2657
|
+
});
|
|
2658
|
+
if (!forgotten) {
|
|
2659
|
+
throw new Error(`Could not complete every cleanup step for "${id}"; its intent was retained`);
|
|
2660
|
+
}
|
|
2661
|
+
return;
|
|
2662
|
+
}
|
|
2663
|
+
if (durable.length) {
|
|
2664
|
+
throw new Error(`Refusing to destroy "${id}" without its durable ownership record; ${durable.length} other attempt(s) remain`);
|
|
2665
|
+
}
|
|
2666
|
+
throw new Error(`Refusing to destroy "${id}" before its legacy ownership can be journaled`);
|
|
2667
|
+
}
|
|
1539
2668
|
try {
|
|
1540
|
-
await this.
|
|
1541
|
-
method: 'DELETE',
|
|
1542
|
-
nullOn: [404],
|
|
1543
|
-
});
|
|
2669
|
+
await this.requestForDestruction(`/apps/${app}/machines/${machineId}?force=true`, { method: 'DELETE', nullOn: [404] }, signal);
|
|
1544
2670
|
}
|
|
1545
2671
|
catch (error) {
|
|
2672
|
+
if (signal.aborted)
|
|
2673
|
+
throw signal.reason;
|
|
1546
2674
|
// The app delete is not attempted: removing the app under a Machine that
|
|
1547
2675
|
// is still alive would hide it. The private addresses are left for the
|
|
1548
2676
|
// retry, which re-reads them from the Machine's metadata.
|
|
@@ -1552,7 +2680,13 @@ class FlyioSandboxProvider {
|
|
|
1552
2680
|
if (this.appPerProject()) {
|
|
1553
2681
|
let appError;
|
|
1554
2682
|
try {
|
|
1555
|
-
await this.
|
|
2683
|
+
const remaining = await this.readAttemptAppState(app, signal);
|
|
2684
|
+
if (remaining.machines.length) {
|
|
2685
|
+
throw new Error(`Refusing to delete Fly app "${app}" while a Machine remains`);
|
|
2686
|
+
}
|
|
2687
|
+
if (!remaining.appGone) {
|
|
2688
|
+
await this.requestForDestruction(`/apps/${app}`, { method: 'DELETE', nullOn: [404] }, signal);
|
|
2689
|
+
}
|
|
1556
2690
|
}
|
|
1557
2691
|
catch (error) {
|
|
1558
2692
|
appError = error;
|
|
@@ -1563,24 +2697,41 @@ class FlyioSandboxProvider {
|
|
|
1563
2697
|
}
|
|
1564
2698
|
// Released even when the app delete failed: the Machine is gone, so a
|
|
1565
2699
|
// retry could no longer read these addresses from its metadata.
|
|
1566
|
-
await this.releasePrivateRoutes(privateRoutes);
|
|
2700
|
+
const routesReleased = await this.releasePrivateRoutes(privateRoutes, signal, this.networkFor(app));
|
|
1567
2701
|
if (appError !== undefined) {
|
|
1568
2702
|
throw new Error(describeDestroyFailure('app delete', id, appError), { cause: appError });
|
|
1569
2703
|
}
|
|
2704
|
+
if (!routesReleased && createAttemptId && this.config.createAttemptJournal) {
|
|
2705
|
+
throw new Error(`Could not release every external Flycast address for "${id}"`);
|
|
2706
|
+
}
|
|
2707
|
+
if (signal.aborted)
|
|
2708
|
+
throw signal.reason;
|
|
2709
|
+
if (createAttemptId)
|
|
2710
|
+
await this.config.createAttemptJournal?.forget(createAttemptId);
|
|
1570
2711
|
return;
|
|
1571
2712
|
}
|
|
2713
|
+
let cleanupFailed = false;
|
|
1572
2714
|
for (const volumeId of volumeIds) {
|
|
1573
2715
|
try {
|
|
1574
|
-
await this.
|
|
1575
|
-
method: 'DELETE',
|
|
1576
|
-
nullOn: [404],
|
|
1577
|
-
});
|
|
2716
|
+
await this.requestForDestruction(`/apps/${app}/volumes/${volumeId}`, { method: 'DELETE', nullOn: [404] }, signal);
|
|
1578
2717
|
}
|
|
1579
2718
|
catch (error) {
|
|
2719
|
+
if (signal.aborted)
|
|
2720
|
+
throw signal.reason;
|
|
2721
|
+
cleanupFailed = true;
|
|
1580
2722
|
logger.warn('Failed to remove Fly volume', { app, volumeId, error });
|
|
1581
2723
|
}
|
|
1582
2724
|
}
|
|
1583
|
-
|
|
2725
|
+
cleanupFailed =
|
|
2726
|
+
!(await this.releasePrivateRoutes(privateRoutes, signal, this.networkFor(app))) ||
|
|
2727
|
+
cleanupFailed;
|
|
2728
|
+
if (cleanupFailed && createAttemptId && this.config.createAttemptJournal) {
|
|
2729
|
+
throw new Error(`Could not complete every cleanup step for "${id}"; its intent was retained`);
|
|
2730
|
+
}
|
|
2731
|
+
if (signal.aborted)
|
|
2732
|
+
throw signal.reason;
|
|
2733
|
+
if (createAttemptId)
|
|
2734
|
+
await this.config.createAttemptJournal?.forget(createAttemptId);
|
|
1584
2735
|
}
|
|
1585
2736
|
// ---------------------------------------------------------------------------
|
|
1586
2737
|
// Templates
|
|
@@ -1659,19 +2810,15 @@ class FlyioSandboxProvider {
|
|
|
1659
2810
|
return templates.removeTemplate(this.templateContext(), templateId);
|
|
1660
2811
|
}
|
|
1661
2812
|
/**
|
|
1662
|
-
* Restores a template into a freshly-created Machine.
|
|
1663
|
-
*
|
|
1664
|
-
* A failure here DESTROYS the Machine. The alternative is to hand back a
|
|
1665
|
-
* sandbox that started cleanly and whose filesystem is not the one that was
|
|
1666
|
-
* asked for — the same silent-wrong-contents failure that makes a missing
|
|
1667
|
-
* template a hard error rather than a fallback to the base image.
|
|
2813
|
+
* Restores a template into a freshly-created Machine. A failure is journaled
|
|
2814
|
+
* by create and settled by the reconciler, never by an inline delete.
|
|
1668
2815
|
* @param app - The Fly app name.
|
|
1669
2816
|
* @param machineId - The Fly Machine id.
|
|
1670
2817
|
* @param templateId - The caller's template identifier.
|
|
1671
2818
|
* @param manifest - The CONTROL-PLANE manifest, whose `capturePaths` bound what is extracted.
|
|
1672
|
-
* @throws {Error} When the restore fails
|
|
2819
|
+
* @throws {Error} When the restore fails.
|
|
1673
2820
|
*/
|
|
1674
|
-
async restoreTemplate(app, machineId, templateId, manifest) {
|
|
2821
|
+
async restoreTemplate(app, machineId, templateId, manifest, owner) {
|
|
1675
2822
|
const ctx = this.templateContext();
|
|
1676
2823
|
const store = templates.requireStore(ctx);
|
|
1677
2824
|
const leaseId = `${machineId}-${Date.now().toString(36)}`.replace(/[^A-Za-z0-9_.-]/g, '_');
|
|
@@ -1679,10 +2826,10 @@ class FlyioSandboxProvider {
|
|
|
1679
2826
|
try {
|
|
1680
2827
|
// The restore runs through exec, so the Machine has to be up. `create`
|
|
1681
2828
|
// launches it but returns as soon as Fly accepts the request.
|
|
1682
|
-
await this.waitForStarted(app, machineId);
|
|
2829
|
+
await this.waitForStarted(app, machineId, owner);
|
|
1683
2830
|
lease = await templates.acquireRestoreLease(ctx, templateId, leaseId);
|
|
1684
2831
|
const url = await store.presignGet(templates.archiveKey(store, templateId), ctx.presignExpirySeconds);
|
|
1685
|
-
const result = await
|
|
2832
|
+
const result = await execCommand(this.rawExec(app, machineId, owner), templates.buildRestoreCommand(manifest.capturePaths, url), { cwd: '/', timeout: ctx.transferTimeoutMs }, '/');
|
|
1686
2833
|
if (result.exitCode !== 0) {
|
|
1687
2834
|
throw new Error(t('codeSandbox.flyio.error.templateRestoreFailed', { templateId, exitCode: String(result.exitCode) }, {
|
|
1688
2835
|
defaultValue: `Restoring template "${templateId}" into Fly Machine ${machineId} failed ` +
|
|
@@ -1690,19 +2837,6 @@ class FlyioSandboxProvider {
|
|
|
1690
2837
|
}));
|
|
1691
2838
|
}
|
|
1692
2839
|
}
|
|
1693
|
-
catch (error) {
|
|
1694
|
-
try {
|
|
1695
|
-
await this.client.request(`/apps/${app}/machines/${machineId}?force=true`, {
|
|
1696
|
-
method: 'DELETE',
|
|
1697
|
-
nullOn: [404],
|
|
1698
|
-
});
|
|
1699
|
-
}
|
|
1700
|
-
catch (cleanupError) {
|
|
1701
|
-
logger.warn('Failed to destroy the Fly Machine whose template restore failed — it will keep ' +
|
|
1702
|
-
'billing until it is reaped', { app, machineId, error: cleanupError });
|
|
1703
|
-
}
|
|
1704
|
-
throw error;
|
|
1705
|
-
}
|
|
1706
2840
|
finally {
|
|
1707
2841
|
if (lease)
|
|
1708
2842
|
await templates.releaseRestoreLease(ctx, lease);
|
|
@@ -1768,15 +2902,32 @@ class FlyioSandboxProvider {
|
|
|
1768
2902
|
: DEFAULT_PROBE_TIMEOUT_MS;
|
|
1769
2903
|
let app = null;
|
|
1770
2904
|
let machineId = null;
|
|
2905
|
+
let ownsProbeApp = false;
|
|
1771
2906
|
let policyPorts;
|
|
1772
2907
|
try {
|
|
1773
2908
|
policyPorts = this.egressAllowedPorts();
|
|
1774
|
-
|
|
1775
|
-
//
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
2909
|
+
// A random name collision is an adoption, never DELETE authority. Retry
|
|
2910
|
+
// fresh names in per-project mode; shared mode keeps its designated app.
|
|
2911
|
+
for (let attempt = 0; attempt < 5; attempt++) {
|
|
2912
|
+
app = this.resolveApp(this.probeProjectId());
|
|
2913
|
+
const ownership = { created: false };
|
|
2914
|
+
// The SAME policy a real sandbox gets (private-service ports included),
|
|
2915
|
+
// but NO Flycast allocation. Accepted app-create ownership is captured
|
|
2916
|
+
// before any later policy/Machine step can fail.
|
|
2917
|
+
await this.ensureApp(app, {
|
|
2918
|
+
privateRoutes: false,
|
|
2919
|
+
requireCreated: this.appPerProject(),
|
|
2920
|
+
ownership,
|
|
2921
|
+
});
|
|
2922
|
+
ownsProbeApp = ownership.created;
|
|
2923
|
+
if (!this.appPerProject() || ownsProbeApp)
|
|
2924
|
+
break;
|
|
2925
|
+
}
|
|
2926
|
+
if (this.appPerProject() && !ownsProbeApp) {
|
|
2927
|
+
throw new Error('Could not allocate a fresh Fly egress probe app name');
|
|
2928
|
+
}
|
|
2929
|
+
if (!app)
|
|
2930
|
+
throw new Error('Could not resolve a Fly egress probe app');
|
|
1780
2931
|
machineId = await this.createProbeMachine(app);
|
|
1781
2932
|
await this.waitForStarted(app, machineId);
|
|
1782
2933
|
const seconds = Math.min(MAX_WAIT_SECONDS, Math.max(1, Math.ceil(timeoutMs / 1000) + PROBE_EXEC_HEADROOM_SECONDS));
|
|
@@ -1804,7 +2955,7 @@ class FlyioSandboxProvider {
|
|
|
1804
2955
|
};
|
|
1805
2956
|
}
|
|
1806
2957
|
finally {
|
|
1807
|
-
await this.destroyProbe(app, machineId);
|
|
2958
|
+
await this.destroyProbe(app, machineId, ownsProbeApp);
|
|
1808
2959
|
}
|
|
1809
2960
|
}
|
|
1810
2961
|
/**
|
|
@@ -1865,15 +3016,17 @@ class FlyioSandboxProvider {
|
|
|
1865
3016
|
return machine.id;
|
|
1866
3017
|
}
|
|
1867
3018
|
/**
|
|
1868
|
-
* Removes the probe Machine and, in per-project mode,
|
|
3019
|
+
* Removes the accepted probe Machine and, in per-project mode, only an app
|
|
3020
|
+
* whose create POST this probe itself had accepted. Adopted/unknown apps KEEP.
|
|
1869
3021
|
*
|
|
1870
3022
|
* Best-effort by design: a cleanup failure must not turn a completed
|
|
1871
3023
|
* observation into an error, and `auto_destroy` plus the bounded init sleep
|
|
1872
3024
|
* already reap the Machine on their own.
|
|
1873
3025
|
* @param app - The probe app, or `null` when it was never resolved.
|
|
1874
|
-
* @param machineId - The probe Machine id, or `null` when
|
|
3026
|
+
* @param machineId - The accepted probe Machine id, or `null` when none was created.
|
|
3027
|
+
* @param appCreated - Accepted app-create provenance for this exact probe.
|
|
1875
3028
|
*/
|
|
1876
|
-
async destroyProbe(app, machineId) {
|
|
3029
|
+
async destroyProbe(app, machineId, appCreated) {
|
|
1877
3030
|
if (!app)
|
|
1878
3031
|
return;
|
|
1879
3032
|
if (machineId) {
|
|
@@ -1887,8 +3040,8 @@ class FlyioSandboxProvider {
|
|
|
1887
3040
|
logger.warn('Failed to remove the Fly egress probe Machine', { app, machineId, error });
|
|
1888
3041
|
}
|
|
1889
3042
|
}
|
|
1890
|
-
// Never in shared-app mode
|
|
1891
|
-
if (!this.appPerProject())
|
|
3043
|
+
// Never in shared-app mode, nor for a colliding/adopted/unknown app.
|
|
3044
|
+
if (!this.appPerProject() || !appCreated)
|
|
1892
3045
|
return;
|
|
1893
3046
|
try {
|
|
1894
3047
|
await this.client.request(`/apps/${app}`, { method: 'DELETE', nullOn: [404] });
|
|
@@ -1950,12 +3103,13 @@ class FlyioSandboxProvider {
|
|
|
1950
3103
|
* @param machineId - The Fly Machine id.
|
|
1951
3104
|
* @returns A transport callback for {@link execCommand}.
|
|
1952
3105
|
*/
|
|
1953
|
-
rawExec(app, machineId) {
|
|
3106
|
+
rawExec(app, machineId, owner) {
|
|
1954
3107
|
return async (command, timeoutSeconds) => {
|
|
1955
3108
|
const deadline = Date.now() + EXEC_PRECONDITION_BUDGET_MS;
|
|
1956
3109
|
for (let attempt = 1;; attempt++) {
|
|
1957
3110
|
try {
|
|
1958
|
-
const
|
|
3111
|
+
const path = `/apps/${app}/machines/${machineId}/exec`;
|
|
3112
|
+
const request = {
|
|
1959
3113
|
method: 'POST',
|
|
1960
3114
|
body: { command, timeout: timeoutSeconds },
|
|
1961
3115
|
// The API's own timeout bounds the command; allow the HTTP request a
|
|
@@ -1965,7 +3119,10 @@ class FlyioSandboxProvider {
|
|
|
1965
3119
|
// exec burst importFiles issues) is Fly API inconsistency under load,
|
|
1966
3120
|
// not a real absence — retry it alongside the default 429/5xx.
|
|
1967
3121
|
retryStatuses: [404],
|
|
1968
|
-
}
|
|
3122
|
+
};
|
|
3123
|
+
const response = owner
|
|
3124
|
+
? await this.requestForAttempt(owner, path, request)
|
|
3125
|
+
: await this.client.request(path, request);
|
|
1969
3126
|
return response ?? {};
|
|
1970
3127
|
}
|
|
1971
3128
|
catch (error) {
|
|
@@ -2111,7 +3268,7 @@ class FlyioSandboxProvider {
|
|
|
2111
3268
|
* @param machineId - The Fly Machine id.
|
|
2112
3269
|
* @throws {Error} When the Machine has not started within the budget.
|
|
2113
3270
|
*/
|
|
2114
|
-
async waitForStarted(app, machineId) {
|
|
3271
|
+
async waitForStarted(app, machineId, owner) {
|
|
2115
3272
|
const totalSeconds = Math.max(1, this.config.startTimeoutSeconds ?? DEFAULT_START_TIMEOUT_SECONDS);
|
|
2116
3273
|
const deadline = Date.now() + totalSeconds * 1000;
|
|
2117
3274
|
let lastState;
|
|
@@ -2121,24 +3278,45 @@ class FlyioSandboxProvider {
|
|
|
2121
3278
|
// enough: a host that does not have the image cached (every first boot
|
|
2122
3279
|
// after an image push) pulls it first, which alone can exceed 60 s.
|
|
2123
3280
|
for (;;) {
|
|
3281
|
+
if (owner) {
|
|
3282
|
+
await this.config.createAttemptJournal?.assertGeneration(owner.attemptId, owner.generation);
|
|
3283
|
+
if (owner.signal.aborted)
|
|
3284
|
+
throw owner.signal.reason;
|
|
3285
|
+
}
|
|
2124
3286
|
const remaining = Math.ceil((deadline - Date.now()) / 1000);
|
|
2125
3287
|
const seconds = Math.max(1, Math.min(remaining, MAX_WAIT_SECONDS));
|
|
2126
3288
|
try {
|
|
2127
3289
|
await this.client.request(`/apps/${app}/machines/${machineId}/wait?state=started&timeout=${seconds}`,
|
|
2128
3290
|
// One attempt: a retry would re-block for the whole round, and the
|
|
2129
3291
|
// state re-read below decides whether another round is warranted.
|
|
2130
|
-
{ attempts: 1, timeoutMs: (seconds + 15) * 1000 });
|
|
3292
|
+
{ attempts: 1, timeoutMs: (seconds + 15) * 1000, signal: owner?.signal });
|
|
3293
|
+
if (owner) {
|
|
3294
|
+
await this.config.createAttemptJournal?.assertGeneration(owner.attemptId, owner.generation);
|
|
3295
|
+
if (owner.signal.aborted)
|
|
3296
|
+
throw owner.signal.reason;
|
|
3297
|
+
}
|
|
2131
3298
|
return;
|
|
2132
3299
|
}
|
|
2133
3300
|
catch (error) {
|
|
3301
|
+
if (owner?.signal.aborted)
|
|
3302
|
+
throw owner.signal.reason;
|
|
2134
3303
|
lastError = error;
|
|
2135
3304
|
// The wait endpoint returning an error is not itself proof the Machine
|
|
2136
3305
|
// failed — re-read the authoritative state before deciding.
|
|
2137
3306
|
const machine = await this.client
|
|
2138
|
-
.request(`/apps/${app}/machines/${machineId}`, {
|
|
3307
|
+
.request(`/apps/${app}/machines/${machineId}`, {
|
|
3308
|
+
nullOn: [404],
|
|
3309
|
+
signal: owner?.signal,
|
|
3310
|
+
})
|
|
2139
3311
|
.catch(() => null);
|
|
2140
|
-
if (machine?.state === 'started')
|
|
3312
|
+
if (machine?.state === 'started') {
|
|
3313
|
+
if (owner) {
|
|
3314
|
+
await this.config.createAttemptJournal?.assertGeneration(owner.attemptId, owner.generation);
|
|
3315
|
+
if (owner.signal.aborted)
|
|
3316
|
+
throw owner.signal.reason;
|
|
3317
|
+
}
|
|
2141
3318
|
return;
|
|
3319
|
+
}
|
|
2142
3320
|
lastState = machine?.state;
|
|
2143
3321
|
// Another round is useful only while the Machine is still on its way
|
|
2144
3322
|
// up (`created` = provisioning/image pull, `starting` = boot). Any
|