@molecule/api-code-sandbox-flyio 1.3.5 → 1.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/provider.js CHANGED
@@ -6,16 +6,16 @@
6
6
  *
7
7
  * @module
8
8
  */
9
- import { createHash } from 'node:crypto';
9
+ import { createHash, randomUUID } from 'node:crypto';
10
10
  import { promisify } from 'node:util';
11
11
  import { gzip } from 'node:zlib';
12
12
  import { getLogger } from '@molecule/api-bond';
13
13
  import { t } from '@molecule/api-i18n';
14
- import { FlyApiClient, FlyApiError, normalizeApiUrl } from './api.js';
14
+ import { FlyApiClient, FlyApiError, normalizeApiUrl, retryDelayMs, } from './api.js';
15
15
  import { buildEgressProbeCommand, DEFAULT_EGRESS_POLICY_NAME, DEFAULT_EGRESS_PROBE_TARGETS, describePublicReach, extractPolicyId, extractPolicyPorts, parseEgressAllowedPorts, parseEgressProbeTargets, verdictForProbeExit, } from './egress.js';
16
16
  import { execCommand, toExecResult } from './exec.js';
17
17
  import { assertPrivateRoutesForEnv, decodePrivateRoutes, DEFAULT_PRIVATE_IP_TYPE, encodePrivateRoutes, flycastHost, mergeEgressPorts, parsePrivateServices, PRIVATE_ROUTES_METADATA_SUFFIX, } from './flycast.js';
18
- import { appNameForProject, mapMachineState, parseSandboxId, toSandboxId } from './ids.js';
18
+ import { appNameForProject, appNameNamespace, mapMachineState, parseSandboxId, toSandboxId, } from './ids.js';
19
19
  import { createTemplateStore, MAX_PRESIGN_EXPIRY_SECONDS } from './storage.js';
20
20
  import * as templates from './templates.js';
21
21
  import { shellQuote } from './utilities.js';
@@ -102,6 +102,8 @@ const DEFAULT_PREVIEW_PORT = 5173;
102
102
  const WORKSPACE_PATH = '/workspace';
103
103
  /** Default volume size in GB when `SandboxConfig.resources.diskMB` is unset. */
104
104
  const DEFAULT_VOLUME_GB = 10;
105
+ /** Exact-identity reads allowed after an ambiguous create before its outcome is declared unknown. */
106
+ const CREATE_RECONCILE_ATTEMPTS = 3;
105
107
  /** Page size for the org-wide Machine listing. */
106
108
  const LIST_PAGE_SIZE = 200;
107
109
  /** Hard cap on pages fetched while listing, so a bad cursor cannot loop forever. */
@@ -204,13 +206,13 @@ const MACHINE_CREATE_ATTEMPTS = 6;
204
206
  * means no Machine was created — unlike a 5xx/timeout, where one might have
205
207
  * been. 412 is safe for the same reason: "volume already claimed by machine
206
208
  * <id>" refuses the create. It is what the create answers in the seconds after
207
- * {@link FlyioSandboxProvider.destroyStaleMachines} removed the Machine that
208
- * held the volume — Fly releases the claim asynchronously, and an immediate
209
- * create lost a production deploy to it (2026-10-14; the identical retry a
209
+ * an explicit destroy released the Machine that held the volume — Fly releases
210
+ * the claim asynchronously, and an immediate create lost a production deploy
211
+ * to it (2026-10-14; the identical retry a
210
212
  * minute later succeeded). Only these two are added on top of the client's
211
213
  * default retryable set; neither retry can double-create.
212
214
  */
213
- const MACHINE_CREATE_RETRY_STATUSES = [404, 412];
215
+ const MACHINE_CREATE_RETRY_STATUSES = [400, 404, 412];
214
216
  /**
215
217
  * What Fly answers when it reached the exec endpoint but the connection to the
216
218
  * Machine's exec agent died before the command ran: `412 {"error":
@@ -300,6 +302,16 @@ const PROBE_EXEC_HEADROOM_SECONDS = 10;
300
302
  */
301
303
  class FlyioSandboxProvider {
302
304
  name = 'flyio';
305
+ /**
306
+ * Machines this provider runs live until destroyed and each app is reachable
307
+ * at its own routable URL (`https://<app>.fly.dev`, plus the shared private
308
+ * network for the platform's front) — the two things a long-lived hosting
309
+ * role needs, which is why this provider is selectable for it.
310
+ */
311
+ hostingCapabilities = {
312
+ longLived: true,
313
+ routable: true,
314
+ };
303
315
  /**
304
316
  * The idle policy a sandbox gets when its own create does not name one — this
305
317
  * provider's configured `autostop` (`FLY_PRODUCTION_AUTOSTOP` for production
@@ -359,8 +371,11 @@ class FlyioSandboxProvider {
359
371
  * Resolves the Fly region for Machines and volumes.
360
372
  * @returns The region code.
361
373
  */
362
- region() {
363
- return this.config.region ?? process.env.FLY_REGION ?? DEFAULT_REGION;
374
+ region(createRegion) {
375
+ // FLY_REGION is the API Machine's own location on Fly, not a tenant
376
+ // placement decision. Only provider configuration or this create's recorded
377
+ // region may choose where new project resources live.
378
+ return createRegion ?? this.config.region ?? DEFAULT_REGION;
364
379
  }
365
380
  /**
366
381
  * Whether each project gets its own Fly app on its own custom 6PN network.
@@ -470,10 +485,51 @@ class FlyioSandboxProvider {
470
485
  * `<app>.internal` already resolves and Flycast would add nothing.
471
486
  * @returns The declared services, or `undefined` when none are configured.
472
487
  */
473
- privateServices() {
488
+ privateServices(createRegion) {
474
489
  if (!this.appPerProject())
475
490
  return undefined;
476
- return (this.config.privateServices ?? parsePrivateServices(process.env.FLY_SANDBOX_PRIVATE_SERVICES));
491
+ const fallback = this.config.privateServices ?? parsePrivateServices(process.env.FLY_SANDBOX_PRIVATE_SERVICES);
492
+ if (!createRegion || createRegion === DEFAULT_REGION)
493
+ return fallback;
494
+ const suffix = createRegion.replace(/-/g, '_').toUpperCase();
495
+ const variable = `FLY_SANDBOX_PRIVATE_SERVICES_${suffix}`;
496
+ const regional = this.config.privateServicesByRegion?.[createRegion] ??
497
+ parsePrivateServices(process.env[variable]);
498
+ if (!regional && fallback?.length) {
499
+ throw new Error(`No Fly private services are configured for create region "${createRegion}"; ` +
500
+ `set ${variable} (or config.privateServicesByRegion.${createRegion}). ` +
501
+ "Refusing to reuse another region's tenant route.");
502
+ }
503
+ return regional ?? fallback;
504
+ }
505
+ /** Fence and cancel every provisioning mutation at its owning attempt. */
506
+ async requestForAttempt(attempt, path, options) {
507
+ if (attempt.signal.aborted)
508
+ throw attempt.signal.reason;
509
+ await this.config.createAttemptJournal?.assertGeneration(attempt.attemptId, attempt.generation);
510
+ if (attempt.signal.aborted)
511
+ throw attempt.signal.reason;
512
+ const result = await this.client.request(path, { ...options, signal: attempt.signal });
513
+ // A response can arrive after ownership was lost. The write may remain
514
+ // unresolved, but a superseded worker must not journal/adopt its outcome.
515
+ if (attempt.signal.aborted)
516
+ throw attempt.signal.reason;
517
+ await this.config.createAttemptJournal?.assertGeneration(attempt.attemptId, attempt.generation);
518
+ return result;
519
+ }
520
+ /** Authorize a DELETE from the committed retired row on its live lock session. */
521
+ async deleteRetired(intent, generation, path, signal) {
522
+ const journal = this.config.createAttemptJournal;
523
+ if (!journal)
524
+ throw new Error('Retired deletion requires its durable create-attempt journal');
525
+ await journal.authorizeDestruction(intent.attemptId, generation);
526
+ return this.requestForDestruction(path, { method: 'DELETE', nullOn: [404] }, signal);
527
+ }
528
+ /** The sole request dispatch for retirement and settlement, always lock-fenced. */
529
+ async requestForDestruction(path, options, signal) {
530
+ if (signal.aborted)
531
+ throw signal.reason;
532
+ return this.client.request(path, { ...options, signal });
477
533
  }
478
534
  /**
479
535
  * Idempotently ensures the Fly app exists, creating it on its own custom 6PN
@@ -495,27 +551,52 @@ class FlyioSandboxProvider {
495
551
  * @returns `routes`: target app name → allocated private address, for
496
552
  * recording in the Machine's metadata (empty when no private services are
497
553
  * declared); `allocated`: the subset of `routes` this call created, which
498
- * the caller releases when the boot fails and the app is NOT discarded
499
- * (an app that pre-existed is kept, and with it the addresses it already
500
- * held — only this call's own allocations are safe to take back).
554
+ * the caller releases when the boot fails. The app itself always survives:
555
+ * another create can adopt it by name while this call is still failing.
501
556
  */
502
557
  async ensureApp(app, options = {}) {
503
558
  const existing = await this.client.request(`/apps/${app}`, { nullOn: [404] });
504
559
  let createdHere = false;
560
+ if (existing && options.requireCreated) {
561
+ return { routes: {}, allocated: {}, created: false };
562
+ }
505
563
  if (!existing) {
506
564
  const body = { name: app, org_slug: this.orgSlug() };
507
565
  const network = this.networkFor(app);
508
566
  if (network)
509
567
  body.network = network;
510
568
  let created = true;
569
+ const attempt = options.attempt;
570
+ const journal = this.config.createAttemptJournal;
571
+ if (attempt && journal) {
572
+ // Persist who is attempting the create BEFORE Fly can accept it. The
573
+ // separate accepted bit below prevents a pending/unknown POST from
574
+ // becoming deletion authority after a crash or reconciliation error.
575
+ await journal.recordAppProvenance(attempt.attemptId, attempt.generation, 'pending');
576
+ }
511
577
  try {
512
- await this.client.request(`/apps`, { method: 'POST', body });
578
+ if (attempt) {
579
+ await this.requestForAttempt(attempt, `/apps`, { method: 'POST', body });
580
+ }
581
+ else {
582
+ await this.client.request(`/apps`, { method: 'POST', body });
583
+ }
584
+ // Accepted POST provenance is established before later policy/route
585
+ // steps can fail, so diagnostic cleanup can still retire its own app.
586
+ if (options.ownership)
587
+ options.ownership.created = true;
588
+ if (attempt && journal) {
589
+ await journal.recordAppProvenance(attempt.attemptId, attempt.generation, 'created');
590
+ }
513
591
  logger.info('Created Fly sandbox app', { app, network: body.network });
514
592
  }
515
593
  catch (error) {
516
594
  // 409/422 means another concurrent boot created the same app first, which
517
595
  // is exactly the outcome we wanted. Anything else is a real failure.
518
596
  if (error instanceof FlyApiError && (error.status === 409 || error.status === 422)) {
597
+ if (attempt && journal) {
598
+ await journal.recordAppProvenance(attempt.attemptId, attempt.generation, 'adopted');
599
+ }
519
600
  logger.debug('Fly sandbox app already exists', { app });
520
601
  created = false;
521
602
  }
@@ -525,45 +606,18 @@ class FlyioSandboxProvider {
525
606
  }
526
607
  createdHere = created;
527
608
  }
528
- try {
529
- if (createdHere && this.config.assignSharedIpv4)
530
- await this.assignSharedIpv4(app);
531
- await this.applyEgressPolicy(app);
532
- if (options.privateRoutes === false)
533
- return { routes: {}, allocated: {}, created: createdHere };
534
- return {
535
- ...(await this.ensurePrivateRoutes(app, Boolean(existing))),
536
- created: createdHere,
537
- };
609
+ if (options.requireCreated && !createdHere) {
610
+ return { routes: {}, allocated: {}, created: false };
538
611
  }
539
- catch (error) {
540
- if (createdHere)
541
- await this.discardCreatedApp(app, {});
542
- throw error;
612
+ if (createdHere && this.config.assignSharedIpv4) {
613
+ await this.assignSharedIpv4(app, options.attempt);
543
614
  }
544
- }
545
- /**
546
- * Deletes an app THIS call created, after a later step of the same create
547
- * failed. Without it every failed create leaves an app behind with its
548
- * addresses and volume: on 2026-09-28, 34 empty `mol-prod-*` apps had piled
549
- * up in production, one per deploy that hit the organization's Machine limit
550
- * (`POST /machines` → 422) after its app was already made. An app that
551
- * existed before the call is never touched — it may hold a working Machine.
552
- * @param app - The app this call created.
553
- * @param privateRoutes - Flycast addresses already allocated for it.
554
- */
555
- async discardCreatedApp(app, privateRoutes) {
556
- try {
557
- await this.client.request(`/apps/${app}`, { method: 'DELETE', nullOn: [404] });
558
- logger.info('Deleted a Fly sandbox app whose create failed', { app });
615
+ await this.applyEgressPolicy(app, options.attempt, options.createRegion);
616
+ if (options.privateRoutes === false) {
617
+ return { routes: {}, allocated: {}, created: createdHere };
559
618
  }
560
- catch (error) {
561
- logger.error('Failed to delete a Fly sandbox app after its create failed — it leaks', {
562
- app,
563
- error,
564
- });
565
- }
566
- await this.releasePrivateRoutes(privateRoutes);
619
+ const routes = await this.ensurePrivateRoutes(app, Boolean(existing), options.attempt, options.createRegion);
620
+ return { ...routes, created: createdHere };
567
621
  }
568
622
  /**
569
623
  * Allocates a Flycast private address on every declared service app, INTO this
@@ -594,37 +648,42 @@ class FlyioSandboxProvider {
594
648
  * the only addresses safe to release when the boot fails after allocation,
595
649
  * since the recorded ones belong to Machines an earlier boot may still run.
596
650
  */
597
- async ensurePrivateRoutes(app, appExisted) {
598
- const services = this.privateServices();
651
+ async ensurePrivateRoutes(app, appExisted, attempt, createRegion) {
652
+ const services = this.privateServices(createRegion);
599
653
  const network = this.networkFor(app);
600
654
  if (!services?.length || !network)
601
655
  return { routes: {}, allocated: {} };
602
656
  const routes = appExisted ? await this.recordedPrivateRoutes(app) : {};
603
657
  // The addresses THIS call allocated, tracked apart from `routes` so a
604
- // failure part-way down the next list can release exactly them. The caller's
605
- // failure path can only discard the APP (`discardCreatedApp(app, {})`), and
606
- // these addresses live on OTHER apps — so without this, an address already
607
- // allocated for an earlier service stays behind on its target app, inside a
608
- // 6PN whose app is gone, and Fly's listing does not report which network an
609
- // address belongs to: nothing could ever find it there to release it.
658
+ // failure part-way down the next list can release exactly them. These
659
+ // addresses live on OTHER apps, so deleting this project's app would not
660
+ // release them; Fly's listing also cannot reconstruct their target network.
610
661
  const allocated = {};
611
- try {
612
- for (const service of services) {
613
- if (routes[service.app])
614
- continue;
615
- const ip = await this.allocatePrivateAddress(service, network);
616
- if (ip) {
617
- routes[service.app] = ip;
618
- allocated[service.app] = ip;
619
- }
662
+ for (const service of services) {
663
+ if (routes[service.app])
664
+ continue;
665
+ if (attempt && this.config.createAttemptJournal) {
666
+ await this.config.createAttemptJournal.recordAddress(attempt.attemptId, {
667
+ targetApp: service.app,
668
+ network,
669
+ owned: false,
670
+ resolved: false,
671
+ });
672
+ }
673
+ const ip = await this.allocatePrivateAddress(service, network, attempt);
674
+ if (attempt && this.config.createAttemptJournal) {
675
+ await this.config.createAttemptJournal.recordAddress(attempt.attemptId, {
676
+ targetApp: service.app,
677
+ network,
678
+ ...(ip ? { ip } : {}),
679
+ owned: Boolean(ip),
680
+ resolved: true,
681
+ });
682
+ }
683
+ if (ip) {
684
+ routes[service.app] = ip;
685
+ allocated[service.app] = ip;
620
686
  }
621
- }
622
- catch (error) {
623
- // Only what this call allocated. An address recorded by an EARLIER boot
624
- // (`appExisted`) is left alone: the app survived this failure and may
625
- // still hold a Machine reaching through that route.
626
- await this.releasePrivateRoutes(allocated);
627
- throw error;
628
687
  }
629
688
  return { routes, allocated };
630
689
  }
@@ -657,50 +716,26 @@ class FlyioSandboxProvider {
657
716
  return {};
658
717
  }
659
718
  /**
660
- * Destroys any Machine already in this project's app before a fresh create.
661
- *
662
- * A create that fails after the Machine exists — a mid-scaffold error, a
663
- * retried queue job — leaves a Machine holding the app's single-attach volume.
664
- * The next create's `POST /machines` then fails with `412 volume already
665
- * claimed by machine <id>`, and the project can never recover on its own. The
666
- * app is one-per-project and holds exactly one sandbox, so any managed Machine
667
- * still here is a stale partial: destroy it (force covers running/suspended) to
668
- * free the volume and make create idempotent per project.
669
- * @param app - The Fly app name.
719
+ * Read and normalize Fly's assignment list once for allocation and settlement.
720
+ * Current Fly returns `{ips: [...]}` and a network object; the older array /
721
+ * `ip_assignments` shapes remain readable for compatibility with saved clients.
670
722
  */
671
- async destroyStaleMachines(app) {
672
- const managedKey = `${this.metadataPrefix}.managed`;
673
- let machines;
674
- try {
675
- machines = await this.client.request(`/apps/${app}/machines`, {
676
- nullOn: [404],
677
- });
678
- }
679
- catch (error) {
680
- // A listing we cannot read is not proof the app is empty. A hidden Machine
681
- // would fail the create with 412 regardless, so surface it and fall
682
- // through rather than pretend the app is clean.
683
- logger.warn('Could not list Machines before create; a stale Machine may still hold the volume', { app, error });
684
- return;
685
- }
686
- for (const machine of Array.isArray(machines) ? machines : []) {
687
- if (machine.config?.metadata?.[managedKey] !== 'true')
688
- continue;
689
- try {
690
- await this.client.request(`/apps/${app}/machines/${machine.id}?force=true`, {
691
- method: 'DELETE',
692
- nullOn: [404],
693
- });
694
- logger.info('Destroyed a stale Machine before re-creating the sandbox', {
695
- app,
696
- machineId: machine.id,
697
- });
698
- }
699
- catch (error) {
700
- logger.warn('Failed to destroy a stale Machine before create; create may fail with ' +
701
- '412 volume already claimed', { app, machineId: machine.id, error });
723
+ async listPrivateAssignments(app, signal) {
724
+ const listed = signal
725
+ ? await this.requestForDestruction(`/apps/${app}/ip_assignments`, {}, signal)
726
+ : await this.client.request(`/apps/${app}/ip_assignments`);
727
+ const assignments = Array.isArray(listed) ? listed : (listed?.ips ?? listed?.ip_assignments);
728
+ if (!assignments)
729
+ return null;
730
+ return assignments.map((assignment) => {
731
+ if (typeof assignment.network === 'string')
732
+ return assignment;
733
+ const network = assignment.network;
734
+ if (!network || network.org_slug !== this.orgSlug()) {
735
+ return { ...assignment, network: undefined };
702
736
  }
703
- }
737
+ return { ...assignment, network: network.name ?? '' };
738
+ });
704
739
  }
705
740
  /**
706
741
  * Allocates ONE Flycast private address on a target app, for this project's
@@ -718,7 +753,7 @@ class FlyioSandboxProvider {
718
753
  * address — an allocation this provider can neither verify nor release later
719
754
  * is untracked state, and the boot is the last moment it can be reported.
720
755
  */
721
- async allocatePrivateAddress(service, network) {
756
+ async allocatePrivateAddress(service, network, attempt) {
722
757
  let assignment;
723
758
  try {
724
759
  // NO org_slug here. The ip_assignments endpoint derives the org from the
@@ -727,25 +762,28 @@ class FlyioSandboxProvider {
727
762
  // slug that app creation REQUIRES is refused here; omitting it returns 200
728
763
  // and allocates the address). app creation still sends org_slug — the two
729
764
  // endpoints genuinely differ.
730
- assignment = await this.client.request(`/apps/${service.app}/ip_assignments`, {
765
+ const request = {
731
766
  method: 'POST',
732
767
  body: {
733
768
  type: this.config.privateIpAssignmentType ?? DEFAULT_PRIVATE_IP_TYPE,
734
769
  network,
735
770
  },
736
- });
771
+ };
772
+ assignment = attempt
773
+ ? await this.requestForAttempt(attempt, `/apps/${service.app}/ip_assignments`, request)
774
+ : await this.client.request(`/apps/${service.app}/ip_assignments`, request);
737
775
  }
738
776
  catch (error) {
739
- // A conflict means an address for this (app, network) pair is already
740
- // there — from a boot that crashed between allocating and recording. The
741
- // ROUTE is what the sandbox needs and it exists, so this is not a boot
742
- // failure; it is an address this provider can no longer release, because
743
- // Fly's listing does not report which network an address belongs to.
777
+ // A conflict can be the duplicate response to an accepted write whose
778
+ // first response was lost. Resolve it by target app AND network, then the
779
+ // caller journals the concrete IP for adoption or later release.
744
780
  if (error instanceof FlyApiError && (error.status === 409 || error.status === 422)) {
745
- logger.warn('A Fly private address for this project’s network already exists on the target app — ' +
746
- 'the route works, but this one cannot be released on destroy. Reconcile with ' +
747
- '`fly ips list --app <target>`.', { target: service.app, network, status: error.status });
748
- return undefined;
781
+ const assignments = await this.listPrivateAssignments(service.app, attempt?.signal);
782
+ const existing = assignments?.find((candidate) => candidate.network === network);
783
+ if (existing?.ip)
784
+ return existing.ip;
785
+ throw new Error(`Fly rejected a private-address allocation for "${service.app}" on "${network}", ` +
786
+ 'but no matching assignment could be resolved', { cause: error });
749
787
  }
750
788
  throw new Error(t('codeSandbox.flyio.error.privateRouteFailed', { app: service.app, network }, {
751
789
  defaultValue: `Could not allocate a Flycast private address on Fly app "${service.app}" for ` +
@@ -771,28 +809,42 @@ class FlyioSandboxProvider {
771
809
  });
772
810
  return assignment.ip;
773
811
  }
812
+ /** Validate the exact IP's current network; metadata alone is not DELETE authority. */
813
+ async privateRouteMatchesNetwork(target, ip, network, signal) {
814
+ if (!network)
815
+ return false;
816
+ const assignments = await this.listPrivateAssignments(target, signal);
817
+ return Boolean(assignments?.some((candidate) => candidate.ip === ip && candidate.network === network));
818
+ }
774
819
  /**
775
- * Releases the Flycast addresses a project's sandbox recorded.
776
- *
777
- * Best-effort: the sandbox is being destroyed either way, and an address left
778
- * behind is a reconciliation item (`fly ips list --app <target>`) rather than
779
- * a failed teardown. It routes into a 6PN that no longer has any app in it.
780
- * @param routes - Target app name → private address, from Machine metadata.
820
+ * Releases only Flycast addresses independently validated for this network.
821
+ * Metadata identifies candidates; it never authorizes a cross-network DELETE.
781
822
  */
782
- async releasePrivateRoutes(routes) {
823
+ async releasePrivateRoutes(routes, signal, network) {
824
+ let released = true;
783
825
  for (const [target, ip] of Object.entries(routes)) {
784
826
  try {
785
- await this.client.request(`/apps/${target}/ip_assignments/${encodeURIComponent(ip)}`, {
786
- method: 'DELETE',
787
- nullOn: [404],
788
- });
827
+ if (!(await this.privateRouteMatchesNetwork(target, ip, network, signal))) {
828
+ released = false;
829
+ logger.warn('Kept a Fly private (Flycast) address with unvalidated network ownership', {
830
+ target,
831
+ ip,
832
+ network,
833
+ });
834
+ continue;
835
+ }
836
+ await this.requestForDestruction(`/apps/${target}/ip_assignments/${encodeURIComponent(ip)}`, { method: 'DELETE', nullOn: [404] }, signal);
789
837
  logger.debug('Released a Fly private (Flycast) address', { target, ip });
790
838
  }
791
839
  catch (error) {
840
+ if (signal.aborted)
841
+ throw signal.reason;
842
+ released = false;
792
843
  logger.warn('Failed to release a Fly private (Flycast) address — it will keep pointing at a 6PN ' +
793
844
  'with no apps in it until it is reconciled with `fly ips list`', { target, ip, error });
794
845
  }
795
846
  }
847
+ return released;
796
848
  }
797
849
  /**
798
850
  * Resolves the ports the egress network policy allows, or `undefined` when no
@@ -811,8 +863,8 @@ class FlyioSandboxProvider {
811
863
  * about a rule with no ports, so accepting `[]` would ship a security
812
864
  * control whose behaviour is a guess.
813
865
  */
814
- egressAllowedPorts() {
815
- return mergeEgressPorts(this.configuredEgressPorts(), this.privateServices());
866
+ egressAllowedPorts(createRegion) {
867
+ return mergeEgressPorts(this.configuredEgressPorts(), this.privateServices(createRegion));
816
868
  }
817
869
  /**
818
870
  * Resolves ONLY the ports the operator configured, before the declared private
@@ -845,8 +897,8 @@ class FlyioSandboxProvider {
845
897
  * @param app - The Fly app name.
846
898
  * @returns Nothing; resolves once the policy is in place (or none was configured).
847
899
  */
848
- async applyEgressPolicy(app) {
849
- const ports = this.egressAllowedPorts();
900
+ async applyEgressPolicy(app, attempt, createRegion) {
901
+ const ports = this.egressAllowedPorts(createRegion);
850
902
  if (!ports)
851
903
  return;
852
904
  const name = this.config.egressPolicyName ?? DEFAULT_EGRESS_POLICY_NAME;
@@ -878,7 +930,18 @@ class FlyioSandboxProvider {
878
930
  selector: { all: true },
879
931
  rules: [{ action: 'allow', direction: 'egress', ports }],
880
932
  };
881
- await this.client.request(`/apps/${app}/network_policies`, { method: 'POST', body: policy });
933
+ if (attempt) {
934
+ await this.requestForAttempt(attempt, `/apps/${app}/network_policies`, {
935
+ method: 'POST',
936
+ body: policy,
937
+ });
938
+ }
939
+ else {
940
+ await this.client.request(`/apps/${app}/network_policies`, {
941
+ method: 'POST',
942
+ body: policy,
943
+ });
944
+ }
882
945
  logger.info('Applied Fly egress network policy', {
883
946
  app,
884
947
  name,
@@ -956,7 +1019,7 @@ class FlyioSandboxProvider {
956
1019
  * rather than failing the boot.
957
1020
  * @param app - The Fly app name.
958
1021
  */
959
- async assignSharedIpv4(app) {
1022
+ async assignSharedIpv4(app, attempt) {
960
1023
  try {
961
1024
  // ONLY `type` — verified against the live API 2026-08-09. Both `region`
962
1025
  // ("region selection is not supported for this IP type", 400) and
@@ -964,37 +1027,51 @@ class FlyioSandboxProvider {
964
1027
  // sandbox app WITHOUT a public IP, so its preview URL never routed and the
965
1028
  // IDE hung on "Loading preview…" forever. The org derives from the app and
966
1029
  // a shared v4 is global (no region).
967
- await this.client.request(`/apps/${app}/ip_assignments`, {
1030
+ const request = {
968
1031
  method: 'POST',
969
- body: {
970
- type: this.config.ipAssignmentType ?? 'shared_v4',
971
- },
972
- });
1032
+ body: { type: this.config.ipAssignmentType ?? 'shared_v4' },
1033
+ };
1034
+ if (attempt)
1035
+ await this.requestForAttempt(attempt, `/apps/${app}/ip_assignments`, request);
1036
+ else
1037
+ await this.client.request(`/apps/${app}/ip_assignments`, request);
973
1038
  }
974
1039
  catch (error) {
1040
+ if (attempt?.signal.aborted)
1041
+ throw attempt.signal.reason;
1042
+ if (attempt) {
1043
+ await this.config.createAttemptJournal?.assertGeneration(attempt.attemptId, attempt.generation);
1044
+ }
975
1045
  logger.warn('Failed to assign a shared IPv4 to the Fly sandbox app — the public preview URL will ' +
976
1046
  'not resolve until an address is allocated', { app, error });
977
1047
  }
978
1048
  }
979
1049
  /**
980
- * Sanitizes a caller-supplied volume name into a valid Fly volume name.
981
- *
982
- * Fly requires `[a-z0-9_]` (lowercase only) and at most 30 characters — it
983
- * rejects anything else with `400 "name only allows lowercase alphanumeric
984
- * characters and underscores with at most 30 characters"`. Callers pass names
985
- * like `mol-<uuid>` (40 chars, hyphens): lowercase, map every other character
986
- * to `_`, and when the result is too long, keep a readable prefix plus a
987
- * deterministic hash suffix so the name stays stable (`findVolume` re-derives
988
- * the same name on the idempotent path) and unique.
989
- * @param name - The caller's volume name.
990
- * @returns A sanitized Fly volume name, `[a-z0-9_]` and <= 30 chars.
1050
+ * Return the readable, sanitized identity shared by attempts for one caller label.
1051
+ * @param name - The caller's volume label.
1052
+ * @returns The `[a-z0-9_]` stem used by {@link volumeName}.
991
1053
  */
992
- volumeName(name) {
1054
+ volumeNameStem(name) {
1055
+ const sanitized = name.toLowerCase().replace(/[^a-z0-9_]/g, '_');
1056
+ return `${sanitized.slice(0, 17)}_`;
1057
+ }
1058
+ /**
1059
+ * Give one create attempt a unique Fly volume name (Fly permits duplicate names).
1060
+ * @param name - The caller's volume label.
1061
+ * @param attemptId - The Machine create attempt's identity.
1062
+ * @returns A sanitized name of at most 30 characters, including the full attempt's digest.
1063
+ */
1064
+ volumeName(name, attemptId) {
1065
+ const digest = createHash('sha256').update(`${name}\0${attemptId}`).digest('hex').slice(0, 12);
1066
+ return `${this.volumeNameStem(name)}${digest}`;
1067
+ }
1068
+ /** The stable pre-attempt name under which durable workspaces were created. */
1069
+ workspaceVolumeName(name) {
993
1070
  const sanitized = name.toLowerCase().replace(/[^a-z0-9_]/g, '_');
994
1071
  if (sanitized.length <= 30)
995
1072
  return sanitized;
996
- const hash = createHash('sha256').update(name).digest('hex').slice(0, 12);
997
- return `${sanitized.slice(0, 17)}_${hash}`;
1073
+ const digest = createHash('sha256').update(name).digest('hex').slice(0, 12);
1074
+ return `${sanitized.slice(0, 17)}_${digest}`;
998
1075
  }
999
1076
  /**
1000
1077
  * Idempotently ensures a volume exists in the app and returns its id.
@@ -1004,28 +1081,138 @@ class FlyioSandboxProvider {
1004
1081
  * size are all known — rather than through the core's optional
1005
1082
  * `createVolume(name)`; see the module `@remarks`.
1006
1083
  * @param app - The Fly app name.
1007
- * @param name - The caller's volume name.
1084
+ * @param flyName - This create attempt's unique, already-sanitized volume name.
1008
1085
  * @param sizeGB - Size to create the volume with, in whole GB.
1009
- * @returns The Fly volume id.
1086
+ * @param compute - The exact guest sent with the Machine create.
1087
+ * @returns The Fly volume id and whether this call created the volume.
1010
1088
  */
1011
- async ensureVolume(app, name, sizeGB) {
1012
- const flyName = this.volumeName(name);
1013
- const existing = await this.findVolume(app, flyName);
1014
- if (existing)
1015
- return existing.id;
1016
- const created = await this.client.request(`/apps/${app}/volumes`, {
1089
+ async ensureVolume(app, projectId, name, attempt, sizeGB, compute, region) {
1090
+ const journal = this.config.createAttemptJournal;
1091
+ const bound = await journal?.findWorkspace(projectId, name);
1092
+ if (bound?.id) {
1093
+ const volumes = await this.client.request(`/apps/${app}/volumes`, {
1094
+ signal: attempt.signal,
1095
+ });
1096
+ const existing = (volumes ?? []).find((volume) => volume.id === bound.id && volume.state !== 'destroyed' && volume.state !== 'destroying');
1097
+ if (!existing) {
1098
+ throw new Error(`Durable workspace volume "${bound.id}" cannot be read; refusing to replace it`);
1099
+ }
1100
+ await journal?.recordVolume(attempt.attemptId, {
1101
+ ...bound,
1102
+ id: existing.id,
1103
+ logicalName: name,
1104
+ workspace: true,
1105
+ resolved: true,
1106
+ createdByAttemptId: undefined,
1107
+ adoptedByAttemptId: attempt.attemptId,
1108
+ });
1109
+ return { id: existing.id, name: bound.name, workspace: true };
1110
+ }
1111
+ // Pre-attempt releases used a deterministic name. Adopt that legacy volume
1112
+ // by ID once; all later recreation uses the durable binding above.
1113
+ const durableName = this.workspaceVolumeName(name);
1114
+ const existing = await this.findVolume(app, durableName, attempt.signal);
1115
+ if (existing) {
1116
+ await journal?.recordVolume(attempt.attemptId, {
1117
+ id: existing.id,
1118
+ name: durableName,
1119
+ logicalName: name,
1120
+ workspace: true,
1121
+ resolved: true,
1122
+ adoptedByAttemptId: attempt.attemptId,
1123
+ });
1124
+ return { id: existing.id, name: durableName, workspace: true };
1125
+ }
1126
+ const attemptName = this.volumeName(name, attempt.attemptId);
1127
+ await journal?.recordVolume(attempt.attemptId, {
1128
+ name: attemptName,
1129
+ logicalName: name,
1130
+ workspace: true,
1131
+ resolved: false,
1132
+ createdByAttemptId: attempt.attemptId,
1133
+ });
1134
+ try {
1135
+ const created = await this.createFreshVolume(app, attemptName, attempt, sizeGB, compute, region);
1136
+ await journal?.recordVolume(attempt.attemptId, {
1137
+ id: created.id,
1138
+ name: attemptName,
1139
+ logicalName: name,
1140
+ workspace: false,
1141
+ resolved: true,
1142
+ createdByAttemptId: created.accepted ? attempt.attemptId : undefined,
1143
+ adoptedByAttemptId: created.accepted ? undefined : attempt.attemptId,
1144
+ });
1145
+ return { id: created.id, name: attemptName, workspace: false };
1146
+ }
1147
+ catch (error) {
1148
+ if (error instanceof FlyApiError &&
1149
+ error.method.toUpperCase() === 'POST' &&
1150
+ error.path === `/apps/${app}/volumes` &&
1151
+ error.status >= 400 &&
1152
+ error.status < 500 &&
1153
+ ![409, 422].includes(error.status)) {
1154
+ // A conflict/validation response can describe a write whose earlier
1155
+ // response was lost, so those outcomes stay unresolved. The 429 replay
1156
+ // contract treats rate-limit rejection as definite; other non-conflict
1157
+ // 4xx responses reject this unique attempt name before creating it.
1158
+ await journal?.recordVolume(attempt.attemptId, {
1159
+ name: attemptName,
1160
+ logicalName: name,
1161
+ workspace: false,
1162
+ resolved: true,
1163
+ });
1164
+ }
1165
+ throw error;
1166
+ }
1167
+ }
1168
+ /**
1169
+ * Creates a volume with the expected Machine's guest as its placement hint.
1170
+ * A 429 is safe to replay directly. After a transport failure or 5xx, bounded
1171
+ * lookups reconcile the attempt's unique name; if it stays invisible the create
1172
+ * fails rather than replaying an ambiguous POST that could duplicate the volume.
1173
+ * @param app - The Fly app name.
1174
+ * @param flyName - This create attempt's unique, already-sanitized volume name.
1175
+ * @param sizeGB - Size to create the volume with, in whole GB.
1176
+ * @param compute - The exact guest sent with the Machine create.
1177
+ * @returns The new Fly volume id.
1178
+ */
1179
+ async createFreshVolume(app, flyName, attempt, sizeGB, compute, region) {
1180
+ let adopted = false;
1181
+ const created = await this.requestForAttempt(attempt, `/apps/${app}/volumes`, {
1017
1182
  method: 'POST',
1018
1183
  body: {
1019
1184
  name: flyName,
1020
- region: this.region(),
1185
+ region,
1021
1186
  size_gb: sizeGB,
1022
1187
  encrypted: true,
1188
+ compute,
1189
+ },
1190
+ reconcile: async (error) => {
1191
+ // A 429 is Fly's confirmation that this request was rejected. Every
1192
+ // other retryable outcome is ambiguous: never replay it until a few
1193
+ // name lookups have had a chance to find the volume Fly may have made.
1194
+ if (error.status !== 0 && error.status < 500)
1195
+ return undefined;
1196
+ for (let lookup = 1; lookup <= CREATE_RECONCILE_ATTEMPTS; lookup++) {
1197
+ const existing = await this.findVolume(app, flyName, attempt.signal);
1198
+ if (existing) {
1199
+ adopted = true;
1200
+ return existing;
1201
+ }
1202
+ if (lookup < CREATE_RECONCILE_ATTEMPTS) {
1203
+ await this.client.sleep(retryDelayMs(lookup));
1204
+ }
1205
+ }
1206
+ // Fly permits duplicate volume names. Negative reads therefore cannot
1207
+ // authorize another POST: stop with the outcome unknown and let the
1208
+ // owning app's durable reconciler reclaim anything the write made.
1209
+ throw new Error(`Fly volume create "${flyName}" has an unknown outcome; refusing to replay it`, { cause: error });
1023
1210
  },
1024
1211
  });
1025
1212
  if (!created?.id) {
1026
1213
  throw new Error(t('codeSandbox.flyio.error.volumeCreateFailed', { name: flyName }, { defaultValue: `Fly volume "${flyName}" was created but the API returned no id.` }));
1027
1214
  }
1028
- return created.id;
1215
+ return { id: created.id, accepted: !adopted };
1029
1216
  }
1030
1217
  /**
1031
1218
  * Finds a volume in an app by name, ignoring ones already being destroyed.
@@ -1033,9 +1220,10 @@ class FlyioSandboxProvider {
1033
1220
  * @param flyName - The sanitized Fly volume name.
1034
1221
  * @returns The matching volume, or `null`.
1035
1222
  */
1036
- async findVolume(app, flyName) {
1223
+ async findVolume(app, flyName, signal) {
1037
1224
  const volumes = await this.client.request(`/apps/${app}/volumes`, {
1038
1225
  nullOn: [404],
1226
+ signal,
1039
1227
  });
1040
1228
  if (!Array.isArray(volumes))
1041
1229
  return null;
@@ -1063,7 +1251,21 @@ class FlyioSandboxProvider {
1063
1251
  * @param config - Project id, image, env, volume name, template, labels and resource limits.
1064
1252
  * @returns A `Sandbox` bound to the new Machine.
1065
1253
  */
1066
- async create(config) {
1254
+ /**
1255
+ * Return the collision-free app name for a complete caller/project identity.
1256
+ * @param projectId - The complete project or per-deploy identity.
1257
+ * @returns The DNS-safe, at-most-63-character app name.
1258
+ */
1259
+ appNameForProject(projectId) {
1260
+ return appNameForProject(this.appPrefix(), projectId);
1261
+ }
1262
+ /**
1263
+ * Create, reconcile and boot one uniquely identified Fly Machine attempt.
1264
+ * Automatic cleanup deletes only a volume or exact Machine ID with this attempt's accepted-create provenance. Apps always remain KEEP for deliberate retirement; adopted, inherited, listed, older-writer and ambiguous resources are also KEPT.
1265
+ * @param config - The sandbox image, project, storage and resource configuration.
1266
+ * @returns The running sandbox.
1267
+ */
1268
+ async create(config, signal = new AbortController().signal) {
1067
1269
  // Resolved BEFORE anything is provisioned, so a bad template id costs no app,
1068
1270
  // no volume and no Machine.
1069
1271
  let manifest = null;
@@ -1083,49 +1285,60 @@ class FlyioSandboxProvider {
1083
1285
  // is checked too — the host is unroutable however the value arrives, and
1084
1286
  // that env is exactly where a control plane puts the values it refuses to
1085
1287
  // leave in a Machine config.
1086
- assertPrivateRoutesForEnv({ ...config.env, ...config.selfDeliveredEnv }, this.privateServices());
1288
+ const createRegion = this.region(config.region);
1289
+ assertPrivateRoutesForEnv({ ...config.env, ...config.selfDeliveredEnv }, this.privateServices(createRegion));
1087
1290
  const app = this.resolveApp(config.projectId);
1088
- const { routes: privateRoutes, allocated, created } = await this.ensureApp(app);
1291
+ const createAttemptId = randomUUID();
1292
+ const journal = this.config.createAttemptJournal;
1293
+ // The SQL journal stays closed until pre-journal/provenance-ignoring writers
1294
+ // have drained. No durable row or Fly resource is created while it is closed.
1295
+ await journal?.assertCreateAllowed();
1296
+ await journal?.begin({
1297
+ attemptId: createAttemptId,
1298
+ projectId: config.projectId,
1299
+ ownerProjectId: String(config.labels?.['molecule.production'] ?? config.projectId),
1300
+ appName: app,
1301
+ });
1302
+ const attempt = { attemptId: createAttemptId, generation: 1, signal };
1089
1303
  try {
1090
- return await this.createInApp(app, privateRoutes, config, manifest);
1304
+ const { routes: privateRoutes } = await this.ensureApp(app, {
1305
+ attempt,
1306
+ createRegion,
1307
+ });
1308
+ const sandbox = await this.createInApp(app, privateRoutes, config, manifest, attempt);
1309
+ if (signal.aborted)
1310
+ throw signal.reason;
1311
+ await journal?.assertGeneration(attempt.attemptId, attempt.generation);
1312
+ await journal?.markState(createAttemptId, 'serving', sandbox.id);
1313
+ return sandbox;
1091
1314
  }
1092
1315
  catch (error) {
1093
- if (created && this.appPerProject()) {
1094
- // The whole app goes, addresses included — discardCreatedApp releases
1095
- // every route it was handed (a created app holds only this call's).
1096
- await this.discardCreatedApp(app, privateRoutes);
1097
- }
1098
- else {
1099
- // The app SURVIVES this failure (it pre-existed, or it is the shared
1100
- // app), so the addresses this call allocated would otherwise stay
1101
- // behind on their target apps with nothing recording them: the
1102
- // Machine that would have carried the metadata was never created, and
1103
- // Fly's listing cannot say which network an address belongs to. Take
1104
- // back exactly this call's own allocations — never the routes an
1105
- // earlier boot recorded, whose Machines may still be running.
1106
- await this.releasePrivateRoutes(allocated);
1107
- }
1316
+ // Create has NO deletion path. The durable attempt is settled by the
1317
+ // separate reconciler after this failure (or after its abandonment window).
1318
+ await journal?.markState(createAttemptId, 'failed').catch((journalError) => {
1319
+ logger.warn('Could not mark a Fly create attempt failed', {
1320
+ app,
1321
+ createAttemptId,
1322
+ error: journalError,
1323
+ });
1324
+ });
1108
1325
  throw error;
1109
1326
  }
1110
1327
  }
1111
1328
  /**
1112
- * The part of {@link create} that runs once the app exists: clear stale
1113
- * Machines, provision the volume, create the Machine and wait for it.
1329
+ * The part of {@link create} that provisions the volume, creates the Machine and waits for it.
1114
1330
  * @param app - The app.
1115
1331
  * @param privateRoutes - Flycast addresses allocated for the app.
1116
1332
  * @param config - The sandbox config.
1117
1333
  * @param manifest - The template manifest, when creating from a template.
1118
1334
  * @returns The running sandbox.
1119
1335
  */
1120
- async createInApp(app, privateRoutes, config, manifest) {
1121
- // Clear any stale Machine from a prior failed/partial create before
1122
- // provisioning — otherwise it still holds the app's single-attach volume and
1123
- // `POST /machines` fails with `412 volume already claimed`. Makes create
1124
- // idempotent so a retried queue job (or a user re-clicking Start) recovers.
1125
- await this.destroyStaleMachines(app);
1336
+ async createInApp(app, privateRoutes, config, manifest, attempt) {
1337
+ const createAttemptId = attempt.attemptId;
1126
1338
  const metadata = {
1127
1339
  [`${this.metadataPrefix}.projectId`]: config.projectId,
1128
1340
  [`${this.metadataPrefix}.managed`]: 'true',
1341
+ [`${this.metadataPrefix}.createAttempt`]: createAttemptId,
1129
1342
  // Recorded so `destroy()` can release exactly the addresses this project
1130
1343
  // allocated. Fly's IP listing does not report which network an address
1131
1344
  // belongs to, so nothing else can reconstruct this.
@@ -1171,34 +1384,112 @@ class FlyioSandboxProvider {
1171
1384
  // the image's idle command and serving nothing.
1172
1385
  if (config.command?.length)
1173
1386
  machineConfig.init = { exec: [...config.command] };
1174
- if (config.volumeName) {
1175
- metadata[`${this.metadataPrefix}.volumeName`] = config.volumeName;
1176
- const sizeGB = config.resources?.diskMB
1387
+ const volumeName = config.volumeName;
1388
+ if (volumeName) {
1389
+ metadata[`${this.metadataPrefix}.volumeName`] = volumeName;
1390
+ const volumeSizeGB = config.resources?.diskMB
1177
1391
  ? Math.max(1, Math.ceil(config.resources.diskMB / 1024))
1178
1392
  : (this.config.defaultVolumeGB ?? DEFAULT_VOLUME_GB);
1179
- const volumeId = await this.ensureVolume(app, config.volumeName, sizeGB);
1393
+ const volume = await this.ensureVolume(app, config.projectId, volumeName, attempt, volumeSizeGB, machineConfig.guest ?? {}, this.region(config.region));
1180
1394
  // The caller may scope the volume to a subtree (e.g. /workspace/<appDir>)
1181
1395
  // so regenerable tooling (node_modules) stays on the fast image-backed
1182
1396
  // rootfs and only durable state pays the volume's small-file IO cost.
1183
- machineConfig.mounts = [{ volume: volumeId, path: config.volumeMountPath ?? WORKSPACE_PATH }];
1397
+ machineConfig.mounts = [{ volume: volume.id, path: config.volumeMountPath ?? WORKSPACE_PATH }];
1184
1398
  }
1185
1399
  const service = this.previewService(config.idleAction);
1186
1400
  if (service)
1187
1401
  machineConfig.services = [service];
1188
- const machine = await this.client.request(`/apps/${app}/machines`, {
1189
- method: 'POST',
1190
- body: { region: this.region(), config: machineConfig },
1191
- // A just-created app propagates to the Machines API asynchronously, so this
1192
- // POST can 404 "app not found" for a few seconds. Retry the 404 (safe: no
1193
- // Machine is created when the app is not found) across the client's backoff.
1194
- attempts: MACHINE_CREATE_ATTEMPTS,
1195
- retryStatuses: MACHINE_CREATE_RETRY_STATUSES,
1196
- });
1402
+ // The POST intent must survive a crash before dispatch. A NULL outcome on
1403
+ // an older row is unknown too; only a new not-started/rejected row proves
1404
+ // that no accepted Machine identity can be hidden by a later empty list.
1405
+ await this.config.createAttemptJournal?.recordMachineCreateOutcome(createAttemptId, attempt.generation, 'pending');
1406
+ let machine;
1407
+ let machineCreateResponseLost = false;
1408
+ try {
1409
+ machine = await this.requestForAttempt(attempt, `/apps/${app}/machines`, {
1410
+ method: 'POST',
1411
+ body: {
1412
+ name: createAttemptId,
1413
+ region: this.region(config.region),
1414
+ config: machineConfig,
1415
+ },
1416
+ // A just-created app propagates to the Machines API asynchronously, so this
1417
+ // POST can 404 "app not found" for a few seconds. Retry the 404 (safe: no
1418
+ // Machine is created when the app is not found) across the client's backoff.
1419
+ attempts: MACHINE_CREATE_ATTEMPTS,
1420
+ retryStatuses: MACHINE_CREATE_RETRY_STATUSES,
1421
+ // A 412 is transient only for the specific released-volume-claim lag.
1422
+ // Placement capacity must reach the caller's bounded retry at once,
1423
+ // with the volume preserved; every other 412 remains a real rejection.
1424
+ retryWhen: (error) => {
1425
+ if (error.status === 400)
1426
+ return /volume not found/i.test(error.body);
1427
+ if (error.status !== 412)
1428
+ return true;
1429
+ return (/volume already claimed by machine/i.test(error.body) ||
1430
+ /volume_placement_capacity/i.test(error.body));
1431
+ },
1432
+ reconcile: async (error) => {
1433
+ // Rate limiting and explicit precondition rejections did not create a
1434
+ // Machine, so their normal bounded replay is safe. A transport/5xx
1435
+ // result is ambiguous: reconcile ONLY this attempt's recorded identity
1436
+ // and never issue another create when the reads cannot find it.
1437
+ if (error.status === 429 || (error.status >= 400 && error.status < 500)) {
1438
+ return undefined;
1439
+ }
1440
+ // A Machine recovered by listing after a lost response is adoption
1441
+ // evidence, not an accepted POST response and never delete authority.
1442
+ machineCreateResponseLost = true;
1443
+ await this.config.createAttemptJournal?.recordMachineCreateOutcome(createAttemptId, attempt.generation, 'unknown');
1444
+ for (let lookup = 1; lookup <= CREATE_RECONCILE_ATTEMPTS; lookup++) {
1445
+ const machines = await this.client.request(`/apps/${app}/machines`, {
1446
+ nullOn: [404],
1447
+ signal: attempt.signal,
1448
+ });
1449
+ const existing = (machines ?? []).find((candidate) => candidate.name === createAttemptId ||
1450
+ candidate.config?.metadata?.[`${this.metadataPrefix}.createAttempt`] ===
1451
+ createAttemptId);
1452
+ if (existing) {
1453
+ await this.config.createAttemptJournal?.recordMachineObservation(createAttemptId, attempt.generation, existing.id);
1454
+ return existing;
1455
+ }
1456
+ if (lookup < CREATE_RECONCILE_ATTEMPTS) {
1457
+ await this.client.sleep(retryDelayMs(lookup));
1458
+ }
1459
+ }
1460
+ throw new Error(`Fly Machine create "${createAttemptId}" in app "${app}" has an unknown outcome; refusing to replay it`, { cause: error });
1461
+ },
1462
+ });
1463
+ }
1464
+ catch (error) {
1465
+ if (attempt.signal.aborted)
1466
+ throw attempt.signal.reason;
1467
+ if (error instanceof FlyApiError &&
1468
+ error.method.toUpperCase() === 'POST' &&
1469
+ error.path === `/apps/${app}/machines` &&
1470
+ error.status >= 400 &&
1471
+ error.status < 500) {
1472
+ try {
1473
+ await this.config.createAttemptJournal?.markMachineCreateFailed(createAttemptId, attempt.generation);
1474
+ }
1475
+ catch (journalError) {
1476
+ logger.warn('Could not record a definitive Fly Machine-create rejection', {
1477
+ app,
1478
+ createAttemptId,
1479
+ error: journalError,
1480
+ });
1481
+ }
1482
+ }
1483
+ throw error;
1484
+ }
1197
1485
  if (!machine?.id) {
1198
1486
  throw new Error(t('codeSandbox.flyio.error.createFailed', { app }, { defaultValue: `Fly Machine creation in app "${app}" returned no Machine id.` }));
1199
1487
  }
1488
+ if (!machineCreateResponseLost) {
1489
+ await this.config.createAttemptJournal?.recordMachineProvenance(createAttemptId, attempt.generation, machine.id);
1490
+ }
1200
1491
  if (manifest && config.templateId) {
1201
- await this.restoreTemplate(app, machine.id, config.templateId, manifest);
1492
+ await this.restoreTemplate(app, machine.id, config.templateId, manifest, attempt);
1202
1493
  // The restore waited for `started` and ran a command in it, so the state
1203
1494
  // read back from the creation response is stale by definition.
1204
1495
  return this.buildSandbox(app, machine.id, 'running');
@@ -1209,35 +1500,7 @@ class FlyioSandboxProvider {
1209
1500
  // `412 failed_precondition: machine not running` until the Machine is up.
1210
1501
  // Wait for `started` here so `create` always hands back a live sandbox, as
1211
1502
  // the JSDoc promises — the template path already does this via restore.
1212
- try {
1213
- await this.waitForStarted(app, machine.id);
1214
- }
1215
- catch (error) {
1216
- // The boot failed after the Machine existed. This call created it and
1217
- // handed it to NOBODY — the caller got a rejection, not a handle — so
1218
- // leaving it behind strands state nothing else ever cleans up: a Machine
1219
- // that reached `started` bills compute until it is reaped, and one that
1220
- // did not keeps the app's single-attach volume claimed (the next
1221
- // create's stale-Machine sweep only runs if this project is retried).
1222
- // The created-app failure path below cannot do this either: it deletes
1223
- // the APP, and `discardCreatedApp` deliberately has no Machine-delete
1224
- // step — an app delete that runs under a live Machine either fails or
1225
- // hides the Machine. Destroy the Machine first (the same order
1226
- // `restoreTemplate`'s failure path and `destroy()` use), then rethrow
1227
- // the boot failure. Best-effort: the boot failure is what the caller
1228
- // must see; a failed cleanup is logged like every other cleanup here.
1229
- try {
1230
- await this.client.request(`/apps/${app}/machines/${machine.id}?force=true`, {
1231
- method: 'DELETE',
1232
- nullOn: [404],
1233
- });
1234
- }
1235
- catch (cleanupError) {
1236
- logger.warn('Failed to destroy the Fly Machine whose boot did not reach started — it will keep ' +
1237
- 'billing (and holding its volume) until it is reaped', { app, machineId: machine.id, error: cleanupError });
1238
- }
1239
- throw error;
1240
- }
1503
+ await this.waitForStarted(app, machine.id, attempt);
1241
1504
  return this.buildSandbox(app, machine.id, 'running');
1242
1505
  }
1243
1506
  /**
@@ -1476,8 +1739,10 @@ class FlyioSandboxProvider {
1476
1739
  'TRUNCATED, so some sandboxes would be missing from the result.',
1477
1740
  }));
1478
1741
  }
1479
- const prefix = `${this.appPrefix()}-`;
1480
- const owned = apps.map((app) => app.name).filter((name) => name?.startsWith(prefix));
1742
+ const namespace = appNameNamespace(this.appPrefix());
1743
+ const owned = apps
1744
+ .map((app) => app.name)
1745
+ .filter((name) => namespace.searchPrefixes.some((prefix) => name?.startsWith(prefix)));
1481
1746
  if (owned.length > FIND_MAX_APPS) {
1482
1747
  throw new Error(t('codeSandbox.flyio.error.findTooManyApps', { count: String(owned.length), max: String(FIND_MAX_APPS) }, {
1483
1748
  defaultValue: `Refusing to enumerate ${owned.length} Fly sandbox apps (cap ${FIND_MAX_APPS}): ` +
@@ -1487,6 +1752,834 @@ class FlyioSandboxProvider {
1487
1752
  }
1488
1753
  return owned;
1489
1754
  }
1755
+ /** Read an existing app's live state; a list 404 after the app read is an error, not absence. */
1756
+ async readAttemptAppState(app, signal) {
1757
+ const [machines, volumes] = await Promise.all([
1758
+ this.requestForDestruction(`/apps/${app}/machines`, { nullOn: [404] }, signal),
1759
+ this.requestForDestruction(`/apps/${app}/volumes`, { nullOn: [404] }, signal),
1760
+ ]);
1761
+ if (Array.isArray(machines) && Array.isArray(volumes)) {
1762
+ return {
1763
+ appGone: false,
1764
+ machines,
1765
+ volumes: volumes.filter((volume) => volume.state !== 'destroyed' && volume.state !== 'destroying'),
1766
+ };
1767
+ }
1768
+ // A Machines/Volumes-list 404 is not proof that a newly-created app is
1769
+ // gone. Confirm the app itself; if it exists, the list read was transient
1770
+ // and responsibility must remain durable.
1771
+ const existing = await this.requestForDestruction(`/apps/${app}`, { nullOn: [404] }, signal);
1772
+ if (existing)
1773
+ throw new Error(`Fly app "${app}" exists but its live-state listing failed`);
1774
+ return { appGone: true, machines: [], volumes: [] };
1775
+ }
1776
+ /** Prove a definitively rejected Machine create has no live consumer before retiring it. */
1777
+ async assertFailedMachineSweepable(intent, signal) {
1778
+ if ((intent.appCreatedByAttemptId === intent.attemptId && intent.appCreateResolved !== true) ||
1779
+ intent.volumes.some((volume) => volume.resolved !== true) ||
1780
+ intent.addresses.some((address) => !address.resolved)) {
1781
+ throw new Error(`Fly create attempt "${intent.attemptId}" still has unresolved ownership`);
1782
+ }
1783
+ const live = await this.readAttemptAppState(intent.appName, signal);
1784
+ if (signal.aborted)
1785
+ throw signal.reason;
1786
+ // Siblings matter only through an actual resource reference below, or at
1787
+ // the later app-deletion boundary. Their mere existence must not strand a
1788
+ // detached, proven-created volume in a shared app.
1789
+ if (live.appGone)
1790
+ return;
1791
+ if (this.appPerProject() && live.machines.length) {
1792
+ throw new Error(`Fly app "${intent.appName}" still has Machines; failed-create sweep is KEEP`);
1793
+ }
1794
+ const volumeIds = new Set(intent.volumes.map((volume) => volume.id).filter((id) => Boolean(id)));
1795
+ for (const machine of live.machines) {
1796
+ if (machine.config?.mounts?.some((mount) => mount.volume && volumeIds.has(mount.volume))) {
1797
+ throw new Error(`Fly Machine "${machine.id}" still references a failed-create volume; sweep is KEEP`);
1798
+ }
1799
+ }
1800
+ for (const volume of live.volumes) {
1801
+ if (volumeIds.has(volume.id) && volume.attached_machine_id) {
1802
+ throw new Error(`Fly volume "${volume.id}" is still attached to "${volume.attached_machine_id}"; sweep is KEEP`);
1803
+ }
1804
+ }
1805
+ }
1806
+ /** Exact durable Machine identity; unlike accepted provenance, sandboxId may be observational. */
1807
+ recordedMachineId(intent) {
1808
+ if (intent.machineId)
1809
+ return intent.machineId;
1810
+ const prefix = `${intent.appName}:`;
1811
+ if (!intent.sandboxId?.startsWith(prefix))
1812
+ return undefined;
1813
+ return intent.sandboxId.slice(prefix.length) || undefined;
1814
+ }
1815
+ /** True when no exact durable Machine ID can settle a possibly-dispatched POST. */
1816
+ hasUnresolvedMachineIdentity(intent) {
1817
+ if (this.recordedMachineId(intent))
1818
+ return false;
1819
+ const legacyUnknown = intent.machineCreateOutcome === undefined &&
1820
+ intent.machineCreateFailed !== true &&
1821
+ !intent.machineId &&
1822
+ !intent.sandboxId &&
1823
+ !intent.machineAdopted;
1824
+ return (legacyUnknown ||
1825
+ intent.machineCreateOutcome === 'pending' ||
1826
+ intent.machineCreateOutcome === 'unknown' ||
1827
+ intent.machineCreateOutcome === 'accepted' ||
1828
+ Boolean(intent.machineId || intent.sandboxId || intent.machineAdopted));
1829
+ }
1830
+ /**
1831
+ * The sole forgetting guard. A listing is never an absence proof: every
1832
+ * durable Machine/volume ID needs this pass's authorized DELETE success/404
1833
+ * or its own exact GET 404. Unresolved identities remain visible as KEEP.
1834
+ */
1835
+ async assertAuthoritativeForget(intent, signal, settled, app) {
1836
+ // A successful non-forced app DELETE (or exact app 404) is authoritative
1837
+ // for every resource contained by that app. External addresses still need
1838
+ // their own settled records below.
1839
+ const containerSettled = app.settled;
1840
+ const machineId = this.recordedMachineId(intent);
1841
+ if (!containerSettled && this.hasUnresolvedMachineIdentity(intent)) {
1842
+ throw new Error(`Fly Machine identity for attempt "${intent.attemptId}" is unresolved; forgetting is KEEP`);
1843
+ }
1844
+ if (!containerSettled && machineId && !settled.machineIds.has(machineId)) {
1845
+ const machine = await this.requestForDestruction(`/apps/${intent.appName}/machines/${machineId}`, { nullOn: [404] }, signal);
1846
+ if (machine) {
1847
+ throw new Error(`Fly Machine "${machineId}" for attempt "${intent.attemptId}" still exists; forgetting is KEEP`);
1848
+ }
1849
+ }
1850
+ for (const volume of containerSettled ? [] : intent.volumes) {
1851
+ if (volume.resolved !== true) {
1852
+ throw new Error(`Fly volume "${volume.id ?? volume.name}" for attempt "${intent.attemptId}" is unresolved; forgetting is KEEP`);
1853
+ }
1854
+ if (!volume.id || settled.volumeIds.has(volume.id))
1855
+ continue;
1856
+ const live = await this.requestForDestruction(`/apps/${intent.appName}/volumes/${volume.id}`, { nullOn: [404] }, signal);
1857
+ if (live) {
1858
+ throw new Error(`Fly volume "${volume.id}" for attempt "${intent.attemptId}" still exists; forgetting is KEEP`);
1859
+ }
1860
+ }
1861
+ if (intent.addresses.some((address) => !address.resolved || address.owned)) {
1862
+ throw new Error(`Fly addresses for attempt "${intent.attemptId}" are not settled; KEEP`);
1863
+ }
1864
+ if (app.required && !app.settled) {
1865
+ throw new Error(`Fly app "${intent.appName}" has no authoritative absence result; KEEP`);
1866
+ }
1867
+ }
1868
+ /**
1869
+ * The ONE automatic deletion/settlement function. Its four invariants are:
1870
+ * (a) it runs only under the owning attempt's lifecycle lock, and re-reads the
1871
+ * durable record and Fly live state inside that lock before EVERY delete;
1872
+ * (b) automatic deletion additionally requires this attempt's accepted-POST
1873
+ * provenance and exact resource ID; adoption is never deletion authority;
1874
+ * (c) unknown outcomes and transient absence retain responsibility. The one
1875
+ * forgetting boundary below requires an authoritative exact-ID result for
1876
+ * every resource: an exact GET 404, or an authorized DELETE success/404. A
1877
+ * listing, empty listing, NULL ID or "no recorded resources" proves no
1878
+ * absence; an unresolved identity keeps the row and reports KEEP. Durable
1879
+ * addresses are released before app deletion/forgetting; and (d) cancellation
1880
+ * plus the attempt's current generation fences every Fly mutation. A stale,
1881
+ * serving or unreadable attempt therefore cannot delete anything or be forgotten.
1882
+ */
1883
+ async settleCreateAttempt(snapshot, signal, generation, options = {}) {
1884
+ const automatic = options.automatic ?? false;
1885
+ const retire = options.retire ?? false;
1886
+ const failedMachineSweep = options.failedMachineSweep ?? false;
1887
+ const deleteApp = options.deleteApp ?? true;
1888
+ const forget = options.forget ?? true;
1889
+ const appRequired = deleteApp && this.appPerProject();
1890
+ // Automatic reconciliation never deletes an app: a container DELETE can
1891
+ // cascade into any sibling's retained or not-yet-resolved user resources.
1892
+ // An existing app remains a durable KEEP for deliberate retirement.
1893
+ const mayDeleteApp = appRequired && !automatic;
1894
+ const journal = this.config.createAttemptJournal;
1895
+ if (!journal)
1896
+ throw new Error('A durable create-attempt journal is required for settlement');
1897
+ const fresh = async () => {
1898
+ if (signal.aborted)
1899
+ throw signal.reason;
1900
+ await journal.assertGeneration(snapshot.attemptId, generation);
1901
+ const current = await journal.get(snapshot.attemptId);
1902
+ if (signal.aborted)
1903
+ throw signal.reason;
1904
+ if (!current || (current.generation ?? 1) !== generation) {
1905
+ throw new Error(`Fly create attempt "${snapshot.attemptId}" was superseded`);
1906
+ }
1907
+ if (current.state !== 'failed' && current.state !== 'abandoned') {
1908
+ throw new Error(`Fly create attempt "${snapshot.attemptId}" is no longer settleable`);
1909
+ }
1910
+ if ((options.expectedAppName && current.appName !== options.expectedAppName) ||
1911
+ (options.expectedMachineId && this.recordedMachineId(current) !== options.expectedMachineId)) {
1912
+ throw new Error(`Fly create attempt "${snapshot.attemptId}" changed outside the caller-validated scope`);
1913
+ }
1914
+ return current;
1915
+ };
1916
+ const ownMachine = (machines, intent) => {
1917
+ // Once an exact ID is durable, a same-name Machine is potentially a
1918
+ // replacement and must never inherit that identity. machineId may be an
1919
+ // observation; only machineCreatedByAttemptId grants automatic authority.
1920
+ const recordedId = this.recordedMachineId(intent);
1921
+ if (recordedId)
1922
+ return machines.find((machine) => machine.id === recordedId);
1923
+ if (options.recordedResourcesOnly)
1924
+ return undefined;
1925
+ const direct = machines.find((machine) => machine.name === intent.attemptId ||
1926
+ machine.config?.metadata?.[`${this.metadataPrefix}.createAttempt`] === intent.attemptId);
1927
+ if (direct || !intent.attemptId.startsWith('legacy:') || !intent.sandboxId)
1928
+ return direct;
1929
+ const prefix = `${intent.appName}:`;
1930
+ if (!intent.sandboxId.startsWith(prefix))
1931
+ return undefined;
1932
+ const sandboxMachineId = intent.sandboxId.slice(prefix.length);
1933
+ return machines.find((machine) => machine.id === sandboxMachineId);
1934
+ };
1935
+ const adoptLegacyResources = async (machine) => {
1936
+ if (!intent.attemptId.startsWith('legacy:'))
1937
+ return;
1938
+ const metadata = machine.config?.metadata ?? {};
1939
+ const logicalName = metadata[`${this.metadataPrefix}.volumeName`];
1940
+ for (const mount of machine.config?.mounts ?? []) {
1941
+ if (!mount.volume || intent.volumes.some((volume) => volume.id === mount.volume))
1942
+ continue;
1943
+ const listed = live.volumes.find((volume) => volume.id === mount.volume);
1944
+ await journal.recordVolume(intent.attemptId, {
1945
+ id: mount.volume,
1946
+ name: listed?.name ?? mount.volume,
1947
+ ...(logicalName ? { logicalName } : {}),
1948
+ workspace: false,
1949
+ resolved: true,
1950
+ adoptedByAttemptId: intent.attemptId,
1951
+ });
1952
+ }
1953
+ const routes = decodePrivateRoutes(metadata[`${this.metadataPrefix}.${PRIVATE_ROUTES_METADATA_SUFFIX}`]);
1954
+ const network = this.networkFor(intent.appName);
1955
+ for (const [targetApp, ip] of Object.entries(routes)) {
1956
+ // A durable exact binding for the intended network is authoritative.
1957
+ // Otherwise validate the IP independently; copied metadata is only a hint.
1958
+ if (intent.addresses.some((address) => address.targetApp === targetApp &&
1959
+ address.network === network &&
1960
+ address.ip === ip &&
1961
+ address.resolved)) {
1962
+ continue;
1963
+ }
1964
+ const validated = await this.privateRouteMatchesNetwork(targetApp, ip, network, signal);
1965
+ await journal.recordAddress(intent.attemptId, {
1966
+ targetApp,
1967
+ network: network ?? intent.appName,
1968
+ ip,
1969
+ owned: validated,
1970
+ resolved: validated,
1971
+ });
1972
+ }
1973
+ };
1974
+ const assertObservedMachineUnreferenced = async (intent, machineId) => {
1975
+ if (intent.machineCreatedByAttemptId === intent.attemptId)
1976
+ return;
1977
+ const siblings = await journal.forApp(intent.appName);
1978
+ if (signal.aborted)
1979
+ throw signal.reason;
1980
+ const conflict = siblings.find((candidate) => candidate.attemptId !== intent.attemptId &&
1981
+ candidate.state !== 'abandoned' &&
1982
+ this.recordedMachineId(candidate) === machineId);
1983
+ if (conflict) {
1984
+ throw new Error(`Fly Machine "${machineId}" is still referenced by ${conflict.state} attempt "${conflict.attemptId}"; observed retirement is KEEP`);
1985
+ }
1986
+ };
1987
+ const activeStates = [
1988
+ 'creating',
1989
+ 'starting',
1990
+ 'started',
1991
+ 'restarting',
1992
+ 'updating',
1993
+ 'replacing',
1994
+ 'migrated',
1995
+ ];
1996
+ const hasActiveSibling = async (intent) => {
1997
+ if (signal.aborted)
1998
+ throw signal.reason;
1999
+ const siblings = await journal.forApp(intent.appName);
2000
+ if (signal.aborted)
2001
+ throw signal.reason;
2002
+ return siblings.some((candidate) => {
2003
+ if (candidate.attemptId === intent.attemptId)
2004
+ return false;
2005
+ if (candidate.state === 'serving')
2006
+ return true;
2007
+ // Every expired sibling in this set is independently reclaimable and
2008
+ // will be retired under its own lock in this pass. The current app lock
2009
+ // fences it now; treating the stale row as live deadlocks both rows.
2010
+ return (candidate.state === 'creating' && !options.expiredCreatingIds?.has(candidate.attemptId));
2011
+ });
2012
+ };
2013
+ const readRecordedMachine = async (current) => {
2014
+ const id = this.recordedMachineId(current);
2015
+ if (!id)
2016
+ return undefined;
2017
+ return this.requestForDestruction(`/apps/${current.appName}/machines/${id}`, { nullOn: [404] }, signal);
2018
+ };
2019
+ const releasedMachineIds = new Set();
2020
+ let intent = await fresh();
2021
+ let live = await this.readAttemptAppState(intent.appName, signal);
2022
+ const observed = ownMachine(options.observedMachines ?? [], intent);
2023
+ if (observed) {
2024
+ // destroyApp read this metadata before waiting for the attempt lock. Save
2025
+ // that observation now; a later list omission cannot erase ownership.
2026
+ intent = await fresh();
2027
+ await adoptLegacyResources(observed);
2028
+ intent = await fresh();
2029
+ }
2030
+ // A list omission cannot erase a durable ID. Read that exact resource;
2031
+ // 403/5xx keeps the row, 404 is authoritative, and a live result follows
2032
+ // the same state/provenance checks as a listed Machine.
2033
+ const machine = ownMachine(live.machines, intent) ??
2034
+ (!forget || mayDeleteApp ? undefined : ((await readRecordedMachine(intent)) ?? undefined));
2035
+ if (machine && !this.recordedMachineId(intent)) {
2036
+ // A name/metadata match recovers an exact identity as observation only.
2037
+ // It deliberately does not gain automatic deletion provenance.
2038
+ await journal.recordMachineObservation(intent.attemptId, generation, machine.id);
2039
+ intent = await fresh();
2040
+ }
2041
+ if (automatic &&
2042
+ machine &&
2043
+ (intent.machineCreatedByAttemptId !== intent.attemptId || intent.machineId !== machine.id)) {
2044
+ throw new Error(`Fly Machine "${machine.id}" for attempt "${intent.attemptId}" has no matching accepted-POST provenance; KEEP`);
2045
+ }
2046
+ if (failedMachineSweep && machine) {
2047
+ throw new Error(`Fly Machine "${machine.id}" appeared during failed-create sweep; KEEP`);
2048
+ }
2049
+ if (!retire &&
2050
+ machine &&
2051
+ (activeStates.includes(machine.state ?? 'created') ||
2052
+ intent.volumes.some((volume) => volume.workspace))) {
2053
+ return false;
2054
+ }
2055
+ if (machine) {
2056
+ // Re-read immediately before the destructive decision, not from the lock-wait snapshot.
2057
+ intent = await fresh();
2058
+ live = await this.readAttemptAppState(intent.appName, signal);
2059
+ const current = ownMachine(live.machines, intent) ??
2060
+ (!forget || mayDeleteApp ? undefined : ((await readRecordedMachine(intent)) ?? undefined));
2061
+ if (failedMachineSweep && current) {
2062
+ throw new Error(`Fly Machine "${current.id}" appeared during failed-create sweep; KEEP`);
2063
+ }
2064
+ if (!retire &&
2065
+ current &&
2066
+ (activeStates.includes(current.state ?? 'created') ||
2067
+ intent.volumes.some((volume) => volume.workspace))) {
2068
+ return false;
2069
+ }
2070
+ if (automatic &&
2071
+ current &&
2072
+ (intent.machineCreatedByAttemptId !== intent.attemptId || intent.machineId !== current.id)) {
2073
+ throw new Error(`Fly Machine "${current.id}" for attempt "${intent.attemptId}" lost accepted-POST provenance before deletion; KEEP`);
2074
+ }
2075
+ if (current) {
2076
+ // Migration 1793275400000 recorded only the synthetic attempt and
2077
+ // sandbox ID. The owned Machine's mounts/routes rebuild the exact-ID
2078
+ // responsibilities before its metadata disappears on deletion.
2079
+ await adoptLegacyResources(current);
2080
+ intent = await fresh();
2081
+ // Observation is not ownership. Recheck every other durable reference
2082
+ // on this same app lock immediately before the irreversible request.
2083
+ await assertObservedMachineUnreferenced(intent, current.id);
2084
+ await this.deleteRetired(intent, generation, `/apps/${intent.appName}/machines/${current.id}?force=true`, signal);
2085
+ releasedMachineIds.add(current.id);
2086
+ }
2087
+ }
2088
+ const releasedVolumeIds = new Set();
2089
+ for (const recorded of intent.volumes) {
2090
+ intent = await fresh();
2091
+ let volumeIntent = intent.volumes.find((candidate) => candidate.name === recorded.name || candidate.id === recorded.id);
2092
+ if (!volumeIntent || (volumeIntent.resolved === true && !volumeIntent.id))
2093
+ continue;
2094
+ live = await this.readAttemptAppState(intent.appName, signal);
2095
+ if (live.appGone)
2096
+ continue;
2097
+ if (options.recordedResourcesOnly && !volumeIntent.id) {
2098
+ throw new Error(`Fly volume "${volumeIntent.name}" for attempt "${intent.attemptId}" has no exact durable ID; refusing discovery during explicit retirement`);
2099
+ }
2100
+ if (!volumeIntent.id) {
2101
+ const matches = live.volumes.filter((candidate) => candidate.name === volumeIntent?.name);
2102
+ if (matches.length > 1) {
2103
+ throw new Error(`Fly volume create "${volumeIntent.name}" has ambiguous matches; an explicit ID is required`);
2104
+ }
2105
+ const discovered = matches[0];
2106
+ if (!discovered) {
2107
+ throw new Error(`Fly volume create "${volumeIntent.name}" remains unresolved`);
2108
+ }
2109
+ // A unique name match is an observation only. Persist its exact ID for
2110
+ // deliberate retirement, never accepted-POST/automatic-delete authority.
2111
+ await journal.recordVolume(intent.attemptId, {
2112
+ ...volumeIntent,
2113
+ id: discovered.id,
2114
+ resolved: true,
2115
+ createdByAttemptId: undefined,
2116
+ adoptedByAttemptId: intent.attemptId,
2117
+ });
2118
+ intent = await fresh();
2119
+ volumeIntent = intent.volumes.find((candidate) => candidate.id === discovered.id);
2120
+ }
2121
+ if (automatic &&
2122
+ volumeIntent &&
2123
+ (volumeIntent.createdByAttemptId !== intent.attemptId || volumeIntent.resolved !== true)) {
2124
+ throw new Error(`Fly volume "${volumeIntent.id ?? volumeIntent.name}" for attempt "${intent.attemptId}" has no matching accepted-POST provenance; KEEP`);
2125
+ }
2126
+ if (!volumeIntent?.id || (!retire && volumeIntent.workspace))
2127
+ continue;
2128
+ // ID, attachment, workspace flag and ownership are all re-read under the lock.
2129
+ intent = await fresh();
2130
+ volumeIntent = intent.volumes.find((candidate) => candidate.id === volumeIntent?.id);
2131
+ live = await this.readAttemptAppState(intent.appName, signal);
2132
+ const volume = live.volumes.find((candidate) => candidate.id === volumeIntent?.id);
2133
+ if (failedMachineSweep && volumeIntent?.id) {
2134
+ const referencing = live.machines.find((machine) => machine.config?.mounts?.some((mount) => mount.volume === volumeIntent?.id));
2135
+ if (referencing || volume?.attached_machine_id) {
2136
+ throw new Error(`Fly volume "${volumeIntent.id}" gained a Machine reference during sweep; KEEP`);
2137
+ }
2138
+ }
2139
+ if (automatic &&
2140
+ volumeIntent &&
2141
+ (volumeIntent.createdByAttemptId !== intent.attemptId || volumeIntent.resolved !== true)) {
2142
+ throw new Error(`Fly volume "${volumeIntent.id ?? volumeIntent.name}" for attempt "${intent.attemptId}" lost accepted-POST provenance before deletion; KEEP`);
2143
+ }
2144
+ if (!volumeIntent?.id || (!retire && volumeIntent.workspace) || volume?.attached_machine_id) {
2145
+ continue;
2146
+ }
2147
+ // A list omission is not deletion. The durable exact ID is sufficient
2148
+ // capability for an idempotent DELETE; success/404, never the list,
2149
+ // authorizes forgetting it. Fly still rejects a really attached volume.
2150
+ const volumeId = volumeIntent.id;
2151
+ await this.deleteRetired(intent, generation, `/apps/${intent.appName}/volumes/${volumeId}`, signal);
2152
+ releasedVolumeIds.add(volumeId);
2153
+ }
2154
+ intent = await fresh();
2155
+ live = await this.readAttemptAppState(intent.appName, signal);
2156
+ if (intent.addresses.length &&
2157
+ (live.machines.some((candidate) => candidate.id !== machine?.id) ||
2158
+ (await hasActiveSibling(intent)))) {
2159
+ return false;
2160
+ }
2161
+ for (const recorded of intent.addresses) {
2162
+ intent = await fresh();
2163
+ let address = intent.addresses.find((candidate) => candidate.targetApp === recorded.targetApp && candidate.network === recorded.network);
2164
+ if (!address || (address.resolved && !address.owned))
2165
+ continue;
2166
+ if (options.recordedResourcesOnly && (!address.owned || !address.ip)) {
2167
+ throw new Error(`Fly address for attempt "${intent.attemptId}" on "${address.targetApp}" has no exact owned IP; refusing discovery during explicit retirement`);
2168
+ }
2169
+ if (!address.resolved) {
2170
+ // Only an unresolved create needs discovery. A known owned IP is its
2171
+ // own DELETE capability, even when the target app no longer exists.
2172
+ const assignments = await this.listPrivateAssignments(address.targetApp, signal);
2173
+ if (!assignments) {
2174
+ throw new Error(`Could not resolve Fly addresses on "${address.targetApp}"`);
2175
+ }
2176
+ const assignment = assignments.find((candidate) => candidate.network === address?.network);
2177
+ if (!assignment?.ip) {
2178
+ throw new Error(`Fly address allocation for attempt "${intent.attemptId}" is unresolved`);
2179
+ }
2180
+ await journal.recordAddress(intent.attemptId, {
2181
+ ...address,
2182
+ ip: assignment.ip,
2183
+ owned: true,
2184
+ resolved: true,
2185
+ });
2186
+ intent = await fresh();
2187
+ address = intent.addresses.find((candidate) => candidate.targetApp === recorded.targetApp && candidate.network === recorded.network);
2188
+ }
2189
+ if (!address?.owned)
2190
+ continue;
2191
+ if (!address.ip)
2192
+ throw new Error(`Owned Fly address for "${intent.attemptId}" has no IP`);
2193
+ // The durable exact IP, not list presence, is the release capability.
2194
+ // A retry after a lost successful DELETE gets 404 and completes below.
2195
+ intent = await fresh();
2196
+ address = intent.addresses.find((candidate) => candidate.targetApp === recorded.targetApp && candidate.network === recorded.network);
2197
+ if (!address?.owned)
2198
+ continue;
2199
+ if (!address.ip)
2200
+ throw new Error(`Owned Fly address for "${intent.attemptId}" has no IP`);
2201
+ await this.deleteRetired(intent, generation, `/apps/${address.targetApp}/ip_assignments/${encodeURIComponent(address.ip)}`, signal);
2202
+ await journal.recordAddress(intent.attemptId, {
2203
+ ...address,
2204
+ owned: false,
2205
+ resolved: true,
2206
+ });
2207
+ }
2208
+ intent = await fresh();
2209
+ live = await this.readAttemptAppState(intent.appName, signal);
2210
+ const ownedVolumesRemain = intent.volumes.some((volume) => {
2211
+ if (volume.resolved === true && !volume.id)
2212
+ return false;
2213
+ if (live.appGone)
2214
+ return false;
2215
+ if (volume.id)
2216
+ return !releasedVolumeIds.has(volume.id);
2217
+ return live.volumes.some((candidate) => candidate.name === volume.name);
2218
+ });
2219
+ // Exact app 404 proves absence of every contained resource, including an
2220
+ // unresolved volume. External addresses still settle independently below.
2221
+ const unsettledVolume = !live.appGone && intent.volumes.some((volume) => volume.resolved !== true);
2222
+ if (failedMachineSweep && this.appPerProject() && live.machines.length) {
2223
+ throw new Error(`Fly app "${intent.appName}" gained a Machine during sweep; KEEP`);
2224
+ }
2225
+ if (failedMachineSweep && deleteApp && this.appPerProject() && live.volumes.length) {
2226
+ throw new Error(`Fly app "${intent.appName}" is not empty; failed-create sweep is KEEP`);
2227
+ }
2228
+ const listedMachine = ownMachine(live.machines, intent);
2229
+ if (unsettledVolume ||
2230
+ ownedVolumesRemain ||
2231
+ (listedMachine && !releasedMachineIds.has(listedMachine.id))) {
2232
+ return false;
2233
+ }
2234
+ if (intent.addresses.some((address) => !address.resolved || address.owned))
2235
+ return false;
2236
+ if (mayDeleteApp && !live.appGone && (await hasActiveSibling(intent))) {
2237
+ return false;
2238
+ }
2239
+ let appSettled = live.appGone;
2240
+ if (mayDeleteApp && !live.appGone) {
2241
+ // Deliberate retirement alone may delete a container. Addresses above are
2242
+ // settled first; provenance and every Machine are re-read under the lock.
2243
+ intent = await fresh();
2244
+ live = await this.readAttemptAppState(intent.appName, signal);
2245
+ if (failedMachineSweep && (live.machines.length || live.volumes.length)) {
2246
+ throw new Error(`Fly app "${intent.appName}" is not empty before app deletion; KEEP`);
2247
+ }
2248
+ if (automatic &&
2249
+ (intent.appCreatedByAttemptId !== intent.attemptId || intent.appCreateResolved !== true)) {
2250
+ throw new Error(`Fly app "${intent.appName}" for attempt "${intent.attemptId}" has no matching accepted-POST provenance; KEEP`);
2251
+ }
2252
+ if (live.machines.length || ((automatic || !retire) && live.volumes.length))
2253
+ return false;
2254
+ await this.deleteRetired(intent, generation, `/apps/${intent.appName}`, signal);
2255
+ appSettled = true;
2256
+ }
2257
+ if (automatic && appRequired && !appSettled && !this.hasUnresolvedMachineIdentity(intent)) {
2258
+ return false;
2259
+ }
2260
+ if (!forget)
2261
+ return false;
2262
+ intent = await fresh();
2263
+ await this.assertAuthoritativeForget(intent, signal, { machineIds: releasedMachineIds, volumeIds: releasedVolumeIds }, { required: appRequired, settled: appSettled });
2264
+ await journal.forget(intent.attemptId, generation);
2265
+ return true;
2266
+ }
2267
+ /** Lock and settle each eligible attempt independently; there is no lock-free mode. */
2268
+ async reconcileCreateAttempts(options = {}) {
2269
+ const journal = this.config.createAttemptJournal;
2270
+ if (!journal)
2271
+ return { deleted: 0, failed: 0 };
2272
+ const now = options.now ?? Date.now();
2273
+ const timeoutMs = options.createTimeoutMs ?? 15 * 60_000;
2274
+ const cutoff = new Date(now - timeoutMs);
2275
+ const intents = (await journal.reclaimable(cutoff)).filter((intent) => !options.attemptId || intent.attemptId === options.attemptId);
2276
+ const expiredCreatingIds = new Set(intents.filter((intent) => intent.state === 'creating').map((intent) => intent.attemptId));
2277
+ const summary = { deleted: 0, failed: 0 };
2278
+ for (const intent of intents) {
2279
+ try {
2280
+ if (!journal.withLock)
2281
+ throw new Error('Attempt settlement requires its lifecycle lock');
2282
+ const forgotten = await journal.withLock(intent, async (signal, generation) => {
2283
+ // Selection happened before the lock wait. Re-read state and expiry;
2284
+ // only failed/expired attempts may be retired by this automatic pass.
2285
+ const current = await journal.get(intent.attemptId);
2286
+ if (!current)
2287
+ return false;
2288
+ if (current.state === 'serving')
2289
+ return false;
2290
+ if (current.state === 'creating') {
2291
+ const expired = (await journal.reclaimable(cutoff)).some((candidate) => candidate.attemptId === intent.attemptId);
2292
+ if (!expired)
2293
+ return false;
2294
+ }
2295
+ if (signal.aborted)
2296
+ throw signal.reason;
2297
+ const failedMachineSweep = current.machineCreateFailed === true;
2298
+ if (failedMachineSweep)
2299
+ await this.assertFailedMachineSweepable(current, signal);
2300
+ if (current.state === 'failed' || current.state === 'creating') {
2301
+ await journal.retire(current.attemptId, generation);
2302
+ }
2303
+ if (signal.aborted)
2304
+ throw signal.reason;
2305
+ const settled = await this.settleCreateAttempt(current, signal, generation, {
2306
+ automatic: true,
2307
+ retire: failedMachineSweep,
2308
+ failedMachineSweep,
2309
+ expiredCreatingIds,
2310
+ });
2311
+ if (failedMachineSweep && !settled) {
2312
+ throw new Error(`Fly failed-create sweep for "${current.attemptId}" remains incomplete`);
2313
+ }
2314
+ return settled;
2315
+ });
2316
+ if (forgotten)
2317
+ summary.deleted++;
2318
+ }
2319
+ catch (error) {
2320
+ // A state transition back to creating/serving is a safe skip, not a cleanup failure.
2321
+ if (!/no longer settleable/.test(error instanceof Error ? error.message : '')) {
2322
+ summary.failed++;
2323
+ logger.warn('Could not reconcile a Fly create attempt; retaining its durable intent', {
2324
+ attemptId: intent.attemptId,
2325
+ app: intent.appName,
2326
+ error,
2327
+ });
2328
+ }
2329
+ }
2330
+ }
2331
+ return summary;
2332
+ }
2333
+ /** Re-read and abandon only while the attempt's current lifecycle lock is live. */
2334
+ async retireCreateAttempt(snapshot, outerSignal, options) {
2335
+ const journal = this.config.createAttemptJournal;
2336
+ if (!journal?.withLock)
2337
+ throw new Error('Attempt settlement requires its lifecycle lock');
2338
+ return journal.withLock(snapshot, async (lockSignal, generation) => {
2339
+ const signal = AbortSignal.any([outerSignal, lockSignal]);
2340
+ if (signal.aborted)
2341
+ throw signal.reason;
2342
+ const current = await journal.get(snapshot.attemptId);
2343
+ if (signal.aborted)
2344
+ throw signal.reason;
2345
+ if (!current)
2346
+ return false;
2347
+ if ((options.expectedAppName && current.appName !== options.expectedAppName) ||
2348
+ (options.expectedMachineId && this.recordedMachineId(current) !== options.expectedMachineId)) {
2349
+ throw new Error(`Fly create attempt "${snapshot.attemptId}" changed outside the caller-validated scope`);
2350
+ }
2351
+ if (current.state !== 'abandoned') {
2352
+ if (signal.aborted)
2353
+ throw signal.reason;
2354
+ await journal.retire(current.attemptId, generation);
2355
+ if (signal.aborted)
2356
+ throw signal.reason;
2357
+ }
2358
+ return this.settleCreateAttempt(current, signal, generation, {
2359
+ retire: true,
2360
+ ...options,
2361
+ });
2362
+ });
2363
+ }
2364
+ /** Journal a pre-journal Machine before its first destructive request. */
2365
+ async journalLegacyMachine(app, machine, fallbackProjectId) {
2366
+ const journal = this.config.createAttemptJournal;
2367
+ if (!journal)
2368
+ throw new Error('Legacy Machine adoption requires its durable journal');
2369
+ const metadata = machine.config?.metadata ?? {};
2370
+ let projectId = metadata[`${this.metadataPrefix}.projectId`] ?? fallbackProjectId;
2371
+ if (!projectId && this.appPerProject()) {
2372
+ const prefix = `${this.appPrefix()}-`;
2373
+ const candidate = app.startsWith(prefix) ? app.slice(prefix.length) : '';
2374
+ if (candidate && appNameForProject(this.appPrefix(), candidate) === app) {
2375
+ projectId = candidate;
2376
+ }
2377
+ }
2378
+ if (!projectId) {
2379
+ throw new Error(`Refusing to destroy legacy Machine "${machine.id}" without its project identity`);
2380
+ }
2381
+ const sandboxId = `${app}:${machine.id}`;
2382
+ const attemptId = `legacy:${sandboxId}`;
2383
+ const existing = await journal.get(attemptId);
2384
+ if (existing) {
2385
+ await journal.markMachineAdopted(attemptId);
2386
+ if (existing.sandboxId) {
2387
+ const adopted = await journal.get(attemptId);
2388
+ if (!adopted)
2389
+ throw new Error(`Could not persist legacy ownership for "${sandboxId}"`);
2390
+ return adopted;
2391
+ }
2392
+ await journal.markState(attemptId, existing.state === 'creating' ? 'serving' : existing.state, sandboxId);
2393
+ const repaired = await journal.get(attemptId);
2394
+ if (!repaired?.sandboxId) {
2395
+ throw new Error(`Could not repair legacy ownership for "${sandboxId}"`);
2396
+ }
2397
+ return repaired;
2398
+ }
2399
+ await journal.begin({
2400
+ attemptId,
2401
+ projectId,
2402
+ ownerProjectId: metadata['molecule.production'] ?? projectId,
2403
+ appName: app,
2404
+ state: 'serving',
2405
+ sandboxId,
2406
+ machineAdopted: true,
2407
+ });
2408
+ const adopted = await journal.get(attemptId);
2409
+ if (!adopted)
2410
+ throw new Error(`Could not persist legacy ownership for "${sandboxId}"`);
2411
+ return adopted;
2412
+ }
2413
+ /** Journal an exclusively owned empty app rather than treating absence as DELETE authority. */
2414
+ async journalLegacyApp(app, projectId) {
2415
+ const journal = this.config.createAttemptJournal;
2416
+ if (!journal)
2417
+ throw new Error('Legacy app adoption requires its durable journal');
2418
+ const attemptId = `legacy:${app}:app`;
2419
+ const existing = await journal.get(attemptId);
2420
+ if (existing)
2421
+ return existing;
2422
+ await journal.begin({ attemptId, projectId, appName: app });
2423
+ await journal.markState(attemptId, 'serving');
2424
+ const adopted = await journal.get(attemptId);
2425
+ if (!adopted)
2426
+ throw new Error(`Could not persist legacy ownership for app "${app}"`);
2427
+ return adopted;
2428
+ }
2429
+ /**
2430
+ * Explicitly destroy an exclusively owned app and its workspace storage.
2431
+ * Recorded external addresses are released before any intent may be forgotten.
2432
+ */
2433
+ async destroyApp(projectId, expectedAppName, signal) {
2434
+ if (!signal)
2435
+ throw new Error('Fly app destruction requires its lifecycle lock signal');
2436
+ if (!this.appPerProject()) {
2437
+ throw new Error('Refusing to delete a shared Fly app as though a caller exclusively owned it');
2438
+ }
2439
+ const app = appNameForProject(this.appPrefix(), projectId);
2440
+ if (expectedAppName !== undefined && expectedAppName !== app) {
2441
+ throw new Error(`Refusing to delete Fly app "${expectedAppName}": its durable identity resolves to "${app}"`);
2442
+ }
2443
+ const listed = await this.requestForDestruction(`/apps/${app}/machines`, { nullOn: [404] }, signal);
2444
+ const machines = Array.isArray(listed) ? listed : [];
2445
+ const routes = {};
2446
+ const attemptIds = new Set();
2447
+ const metadataHints = [];
2448
+ for (const machine of machines) {
2449
+ Object.assign(routes, decodePrivateRoutes(machine.config?.metadata?.[`${this.metadataPrefix}.${PRIVATE_ROUTES_METADATA_SUFFIX}`]));
2450
+ const attemptId = machine.config?.metadata?.[`${this.metadataPrefix}.createAttempt`];
2451
+ if (attemptId) {
2452
+ attemptIds.add(attemptId);
2453
+ metadataHints.push({ attemptId, machineId: machine.id });
2454
+ }
2455
+ }
2456
+ const journal = this.config.createAttemptJournal;
2457
+ if (journal) {
2458
+ const durable = new Map();
2459
+ const expectedMachineIds = new Map();
2460
+ const journaled = await journal.forApp(app);
2461
+ if (signal.aborted)
2462
+ throw signal.reason;
2463
+ for (const intent of journaled)
2464
+ durable.set(intent.attemptId, intent);
2465
+ for (const hint of metadataHints) {
2466
+ const intent = await journal.get(hint.attemptId);
2467
+ if (signal.aborted)
2468
+ throw signal.reason;
2469
+ // Metadata is only a hint. Both the requested app and this listed
2470
+ // Machine's exact durable identity must match before selection.
2471
+ if (intent?.appName === app && this.recordedMachineId(intent) === hint.machineId) {
2472
+ durable.set(intent.attemptId, intent);
2473
+ expectedMachineIds.set(intent.attemptId, hint.machineId);
2474
+ }
2475
+ else if (intent) {
2476
+ throw new Error(`Refusing copied createAttempt metadata for Machine "${hint.machineId}" in app "${app}"`);
2477
+ }
2478
+ }
2479
+ for (const machine of machines) {
2480
+ const createAttemptId = machine.config?.metadata?.[`${this.metadataPrefix}.createAttempt`];
2481
+ const known = [...durable.values()].some((intent) => intent.appName === app &&
2482
+ ((intent.attemptId === createAttemptId &&
2483
+ this.recordedMachineId(intent) === machine.id) ||
2484
+ intent.sandboxId === `${app}:${machine.id}`));
2485
+ if (known)
2486
+ continue;
2487
+ const adopted = await this.journalLegacyMachine(app, machine, projectId);
2488
+ durable.set(adopted.attemptId, adopted);
2489
+ }
2490
+ if (!durable.size) {
2491
+ const adopted = await this.journalLegacyApp(app, projectId);
2492
+ durable.set(adopted.attemptId, adopted);
2493
+ }
2494
+ if (durable.size && !journal.withLock) {
2495
+ throw new Error(`Refusing to retire Fly app "${app}" without its lifecycle lock`);
2496
+ }
2497
+ if (durable.size && journal.withLock) {
2498
+ // Pass one settles every durable address and retires Machines without
2499
+ // deleting the app. No app deletion can bypass another attempt's failed
2500
+ // address release. Pass two deletes the app and forgets settled intents.
2501
+ for (const deleteApp of [false, true]) {
2502
+ for (const snapshot of durable.values()) {
2503
+ const current = await journal.get(snapshot.attemptId);
2504
+ if (signal.aborted)
2505
+ throw signal.reason;
2506
+ if (!current)
2507
+ continue;
2508
+ await this.retireCreateAttempt(current, signal, {
2509
+ deleteApp,
2510
+ forget: deleteApp,
2511
+ observedMachines: machines,
2512
+ expectedAppName: app,
2513
+ expectedMachineId: expectedMachineIds.get(current.attemptId),
2514
+ });
2515
+ }
2516
+ }
2517
+ const retained = (await Promise.all([...durable.keys()].map(async (attemptId) => journal.get(attemptId)))).some(Boolean);
2518
+ if (signal.aborted)
2519
+ throw signal.reason;
2520
+ if (retained)
2521
+ throw new Error(`Fly app "${app}" retirement has not fully settled`);
2522
+ return;
2523
+ }
2524
+ }
2525
+ // Legacy unjournaled apps retain explicit destruction. Never force an app
2526
+ // through a Machine that has no committed retired row: explicitly delete
2527
+ // each observed Machine, release its routes, then re-list before app DELETE.
2528
+ for (const machine of machines) {
2529
+ await this.requestForDestruction(`/apps/${app}/machines/${machine.id}?force=true`, { method: 'DELETE', nullOn: [404] }, signal);
2530
+ }
2531
+ for (const [targetApp, ip] of Object.entries(routes)) {
2532
+ if (!(await this.privateRouteMatchesNetwork(targetApp, ip, this.networkFor(app), signal))) {
2533
+ logger.warn('Kept a Fly private (Flycast) address with unvalidated network ownership', {
2534
+ targetApp,
2535
+ ip,
2536
+ app,
2537
+ });
2538
+ continue;
2539
+ }
2540
+ await this.requestForDestruction(`/apps/${targetApp}/ip_assignments/${encodeURIComponent(ip)}`, { method: 'DELETE', nullOn: [404] }, signal);
2541
+ }
2542
+ const remaining = await this.readAttemptAppState(app, signal);
2543
+ if (remaining.machines.length) {
2544
+ throw new Error(`Refusing to delete Fly app "${app}" while a Machine remains`);
2545
+ }
2546
+ if (!remaining.appGone) {
2547
+ await this.requestForDestruction(`/apps/${app}`, { method: 'DELETE', nullOn: [404] }, signal);
2548
+ }
2549
+ for (const attemptId of attemptIds) {
2550
+ if (signal.aborted)
2551
+ throw signal.reason;
2552
+ await journal?.forget(attemptId);
2553
+ }
2554
+ }
2555
+ /**
2556
+ * Deliberately retire one named shared-app attempt using only exact durable
2557
+ * resource IDs. Unlike legacy destroy(), this never infers ownership from a
2558
+ * Machine name, metadata, sandboxId, a volume name, or an address listing.
2559
+ */
2560
+ async destroyAttempt(attemptId, signal) {
2561
+ if (!signal)
2562
+ throw new Error('Fly attempt destruction requires its lifecycle lock signal');
2563
+ if (this.appPerProject()) {
2564
+ throw new Error('Use explicit per-project app destruction for a per-project Fly attempt');
2565
+ }
2566
+ const journal = this.config.createAttemptJournal;
2567
+ if (!journal)
2568
+ throw new Error('Fly attempt destruction requires its durable journal');
2569
+ const snapshot = await journal.get(attemptId);
2570
+ if (signal.aborted)
2571
+ throw signal.reason;
2572
+ if (!snapshot)
2573
+ return;
2574
+ const forgotten = await this.retireCreateAttempt(snapshot, signal, {
2575
+ deleteApp: false,
2576
+ forget: true,
2577
+ recordedResourcesOnly: true,
2578
+ });
2579
+ if (!forgotten) {
2580
+ throw new Error(`Could not complete exact-resource cleanup for attempt "${attemptId}"; its intent was retained`);
2581
+ }
2582
+ }
1490
2583
  /**
1491
2584
  * Destroys a sandbox.
1492
2585
  *
@@ -1504,15 +2597,19 @@ class FlyioSandboxProvider {
1504
2597
  * other than a 404 (which counts as already gone), naming the failed step and
1505
2598
  * HTTP status, so the caller can retry instead of forgetting a billed app.
1506
2599
  */
1507
- async destroy(id) {
2600
+ async destroy(id, signal) {
2601
+ if (!signal)
2602
+ throw new Error('Fly destruction requires its lifecycle lock signal');
1508
2603
  const { app, machineId } = parseSandboxId(id);
1509
2604
  let volumeIds = [];
1510
2605
  let privateRoutes = {};
2606
+ let createAttemptId;
2607
+ let observedMachine = null;
1511
2608
  try {
1512
- const machine = await this.client.request(`/apps/${app}/machines/${machineId}`, {
1513
- nullOn: [404],
1514
- });
2609
+ const machine = await this.requestForDestruction(`/apps/${app}/machines/${machineId}`, { nullOn: [404] }, signal);
2610
+ observedMachine = machine;
1515
2611
  privateRoutes = decodePrivateRoutes(machine?.config?.metadata?.[`${this.metadataPrefix}.${PRIVATE_ROUTES_METADATA_SUFFIX}`]);
2612
+ createAttemptId = machine?.config?.metadata?.[`${this.metadataPrefix}.createAttempt`];
1516
2613
  if (!this.appPerProject()) {
1517
2614
  volumeIds = (machine?.config?.mounts ?? [])
1518
2615
  .map((mount) => mount.volume)
@@ -1520,19 +2617,60 @@ class FlyioSandboxProvider {
1520
2617
  }
1521
2618
  }
1522
2619
  catch (error) {
2620
+ if (signal.aborted)
2621
+ throw signal.reason;
1523
2622
  // Best-effort metadata gathering; the Machine may already be gone.
1524
2623
  // Destruction proceeds either way — but say so, because an unread
1525
2624
  // metadata block means the private addresses below are NOT released.
1526
2625
  logger.warn('Could not read the Fly Machine before destroy — its volumes and any private (Flycast) ' +
1527
2626
  'addresses it recorded will not be cleaned up', { id, error });
1528
2627
  }
2628
+ const journal = this.config.createAttemptJournal;
2629
+ if (journal) {
2630
+ const durable = await journal.forApp(app);
2631
+ if (signal.aborted)
2632
+ throw signal.reason;
2633
+ // Metadata is only a lookup hint. It selects an attempt when its durable
2634
+ // exact Machine identity also names the caller's ID; copied metadata must
2635
+ // never redirect retirement to a different Machine's attempt.
2636
+ let intent = durable.find((candidate) => candidate.attemptId === createAttemptId &&
2637
+ this.recordedMachineId(candidate) === machineId) ?? durable.find((candidate) => candidate.sandboxId === id);
2638
+ if (!intent && observedMachine) {
2639
+ intent = await this.journalLegacyMachine(app, observedMachine);
2640
+ }
2641
+ if (intent) {
2642
+ const current = await journal.get(intent.attemptId);
2643
+ if (signal.aborted)
2644
+ throw signal.reason;
2645
+ if (!current)
2646
+ throw new Error(`Fly create attempt "${intent.attemptId}" disappeared`);
2647
+ const durableMachineId = this.recordedMachineId(current);
2648
+ if (durableMachineId !== machineId) {
2649
+ throw new Error(`Refusing to retire attempt "${current.attemptId}": caller named Machine "${machineId}" but durable identity is "${durableMachineId ?? 'unresolved'}"`);
2650
+ }
2651
+ const forgotten = await this.retireCreateAttempt(current, signal, {
2652
+ deleteApp: this.appPerProject(),
2653
+ forget: true,
2654
+ observedMachines: observedMachine ? [observedMachine] : [],
2655
+ expectedAppName: app,
2656
+ expectedMachineId: machineId,
2657
+ });
2658
+ if (!forgotten) {
2659
+ throw new Error(`Could not complete every cleanup step for "${id}"; its intent was retained`);
2660
+ }
2661
+ return;
2662
+ }
2663
+ if (durable.length) {
2664
+ throw new Error(`Refusing to destroy "${id}" without its durable ownership record; ${durable.length} other attempt(s) remain`);
2665
+ }
2666
+ throw new Error(`Refusing to destroy "${id}" before its legacy ownership can be journaled`);
2667
+ }
1529
2668
  try {
1530
- await this.client.request(`/apps/${app}/machines/${machineId}?force=true`, {
1531
- method: 'DELETE',
1532
- nullOn: [404],
1533
- });
2669
+ await this.requestForDestruction(`/apps/${app}/machines/${machineId}?force=true`, { method: 'DELETE', nullOn: [404] }, signal);
1534
2670
  }
1535
2671
  catch (error) {
2672
+ if (signal.aborted)
2673
+ throw signal.reason;
1536
2674
  // The app delete is not attempted: removing the app under a Machine that
1537
2675
  // is still alive would hide it. The private addresses are left for the
1538
2676
  // retry, which re-reads them from the Machine's metadata.
@@ -1542,7 +2680,13 @@ class FlyioSandboxProvider {
1542
2680
  if (this.appPerProject()) {
1543
2681
  let appError;
1544
2682
  try {
1545
- await this.client.request(`/apps/${app}`, { method: 'DELETE', nullOn: [404] });
2683
+ const remaining = await this.readAttemptAppState(app, signal);
2684
+ if (remaining.machines.length) {
2685
+ throw new Error(`Refusing to delete Fly app "${app}" while a Machine remains`);
2686
+ }
2687
+ if (!remaining.appGone) {
2688
+ await this.requestForDestruction(`/apps/${app}`, { method: 'DELETE', nullOn: [404] }, signal);
2689
+ }
1546
2690
  }
1547
2691
  catch (error) {
1548
2692
  appError = error;
@@ -1553,24 +2697,41 @@ class FlyioSandboxProvider {
1553
2697
  }
1554
2698
  // Released even when the app delete failed: the Machine is gone, so a
1555
2699
  // retry could no longer read these addresses from its metadata.
1556
- await this.releasePrivateRoutes(privateRoutes);
2700
+ const routesReleased = await this.releasePrivateRoutes(privateRoutes, signal, this.networkFor(app));
1557
2701
  if (appError !== undefined) {
1558
2702
  throw new Error(describeDestroyFailure('app delete', id, appError), { cause: appError });
1559
2703
  }
2704
+ if (!routesReleased && createAttemptId && this.config.createAttemptJournal) {
2705
+ throw new Error(`Could not release every external Flycast address for "${id}"`);
2706
+ }
2707
+ if (signal.aborted)
2708
+ throw signal.reason;
2709
+ if (createAttemptId)
2710
+ await this.config.createAttemptJournal?.forget(createAttemptId);
1560
2711
  return;
1561
2712
  }
2713
+ let cleanupFailed = false;
1562
2714
  for (const volumeId of volumeIds) {
1563
2715
  try {
1564
- await this.client.request(`/apps/${app}/volumes/${volumeId}`, {
1565
- method: 'DELETE',
1566
- nullOn: [404],
1567
- });
2716
+ await this.requestForDestruction(`/apps/${app}/volumes/${volumeId}`, { method: 'DELETE', nullOn: [404] }, signal);
1568
2717
  }
1569
2718
  catch (error) {
2719
+ if (signal.aborted)
2720
+ throw signal.reason;
2721
+ cleanupFailed = true;
1570
2722
  logger.warn('Failed to remove Fly volume', { app, volumeId, error });
1571
2723
  }
1572
2724
  }
1573
- await this.releasePrivateRoutes(privateRoutes);
2725
+ cleanupFailed =
2726
+ !(await this.releasePrivateRoutes(privateRoutes, signal, this.networkFor(app))) ||
2727
+ cleanupFailed;
2728
+ if (cleanupFailed && createAttemptId && this.config.createAttemptJournal) {
2729
+ throw new Error(`Could not complete every cleanup step for "${id}"; its intent was retained`);
2730
+ }
2731
+ if (signal.aborted)
2732
+ throw signal.reason;
2733
+ if (createAttemptId)
2734
+ await this.config.createAttemptJournal?.forget(createAttemptId);
1574
2735
  }
1575
2736
  // ---------------------------------------------------------------------------
1576
2737
  // Templates
@@ -1649,19 +2810,15 @@ class FlyioSandboxProvider {
1649
2810
  return templates.removeTemplate(this.templateContext(), templateId);
1650
2811
  }
1651
2812
  /**
1652
- * Restores a template into a freshly-created Machine.
1653
- *
1654
- * A failure here DESTROYS the Machine. The alternative is to hand back a
1655
- * sandbox that started cleanly and whose filesystem is not the one that was
1656
- * asked for — the same silent-wrong-contents failure that makes a missing
1657
- * template a hard error rather than a fallback to the base image.
2813
+ * Restores a template into a freshly-created Machine. A failure is journaled
2814
+ * by create and settled by the reconciler, never by an inline delete.
1658
2815
  * @param app - The Fly app name.
1659
2816
  * @param machineId - The Fly Machine id.
1660
2817
  * @param templateId - The caller's template identifier.
1661
2818
  * @param manifest - The CONTROL-PLANE manifest, whose `capturePaths` bound what is extracted.
1662
- * @throws {Error} When the restore fails; the Machine is destroyed first.
2819
+ * @throws {Error} When the restore fails.
1663
2820
  */
1664
- async restoreTemplate(app, machineId, templateId, manifest) {
2821
+ async restoreTemplate(app, machineId, templateId, manifest, owner) {
1665
2822
  const ctx = this.templateContext();
1666
2823
  const store = templates.requireStore(ctx);
1667
2824
  const leaseId = `${machineId}-${Date.now().toString(36)}`.replace(/[^A-Za-z0-9_.-]/g, '_');
@@ -1669,10 +2826,10 @@ class FlyioSandboxProvider {
1669
2826
  try {
1670
2827
  // The restore runs through exec, so the Machine has to be up. `create`
1671
2828
  // launches it but returns as soon as Fly accepts the request.
1672
- await this.waitForStarted(app, machineId);
2829
+ await this.waitForStarted(app, machineId, owner);
1673
2830
  lease = await templates.acquireRestoreLease(ctx, templateId, leaseId);
1674
2831
  const url = await store.presignGet(templates.archiveKey(store, templateId), ctx.presignExpirySeconds);
1675
- const result = await ctx.exec(app, machineId, templates.buildRestoreCommand(manifest.capturePaths, url), ctx.transferTimeoutMs);
2832
+ const result = await execCommand(this.rawExec(app, machineId, owner), templates.buildRestoreCommand(manifest.capturePaths, url), { cwd: '/', timeout: ctx.transferTimeoutMs }, '/');
1676
2833
  if (result.exitCode !== 0) {
1677
2834
  throw new Error(t('codeSandbox.flyio.error.templateRestoreFailed', { templateId, exitCode: String(result.exitCode) }, {
1678
2835
  defaultValue: `Restoring template "${templateId}" into Fly Machine ${machineId} failed ` +
@@ -1680,19 +2837,6 @@ class FlyioSandboxProvider {
1680
2837
  }));
1681
2838
  }
1682
2839
  }
1683
- catch (error) {
1684
- try {
1685
- await this.client.request(`/apps/${app}/machines/${machineId}?force=true`, {
1686
- method: 'DELETE',
1687
- nullOn: [404],
1688
- });
1689
- }
1690
- catch (cleanupError) {
1691
- logger.warn('Failed to destroy the Fly Machine whose template restore failed — it will keep ' +
1692
- 'billing until it is reaped', { app, machineId, error: cleanupError });
1693
- }
1694
- throw error;
1695
- }
1696
2840
  finally {
1697
2841
  if (lease)
1698
2842
  await templates.releaseRestoreLease(ctx, lease);
@@ -1758,15 +2902,32 @@ class FlyioSandboxProvider {
1758
2902
  : DEFAULT_PROBE_TIMEOUT_MS;
1759
2903
  let app = null;
1760
2904
  let machineId = null;
2905
+ let ownsProbeApp = false;
1761
2906
  let policyPorts;
1762
2907
  try {
1763
2908
  policyPorts = this.egressAllowedPorts();
1764
- app = this.resolveApp(this.probeProjectId());
1765
- // The SAME policy a real sandbox gets (private-service ports included, so
1766
- // the probe observes the real posture), but NO Flycast allocation: this
1767
- // app is deleted seconds from now, and allocating would leave one
1768
- // unreleasable address per probe on every target app — every 15 minutes.
1769
- await this.ensureApp(app, { privateRoutes: false });
2909
+ // A random name collision is an adoption, never DELETE authority. Retry
2910
+ // fresh names in per-project mode; shared mode keeps its designated app.
2911
+ for (let attempt = 0; attempt < 5; attempt++) {
2912
+ app = this.resolveApp(this.probeProjectId());
2913
+ const ownership = { created: false };
2914
+ // The SAME policy a real sandbox gets (private-service ports included),
2915
+ // but NO Flycast allocation. Accepted app-create ownership is captured
2916
+ // before any later policy/Machine step can fail.
2917
+ await this.ensureApp(app, {
2918
+ privateRoutes: false,
2919
+ requireCreated: this.appPerProject(),
2920
+ ownership,
2921
+ });
2922
+ ownsProbeApp = ownership.created;
2923
+ if (!this.appPerProject() || ownsProbeApp)
2924
+ break;
2925
+ }
2926
+ if (this.appPerProject() && !ownsProbeApp) {
2927
+ throw new Error('Could not allocate a fresh Fly egress probe app name');
2928
+ }
2929
+ if (!app)
2930
+ throw new Error('Could not resolve a Fly egress probe app');
1770
2931
  machineId = await this.createProbeMachine(app);
1771
2932
  await this.waitForStarted(app, machineId);
1772
2933
  const seconds = Math.min(MAX_WAIT_SECONDS, Math.max(1, Math.ceil(timeoutMs / 1000) + PROBE_EXEC_HEADROOM_SECONDS));
@@ -1794,7 +2955,7 @@ class FlyioSandboxProvider {
1794
2955
  };
1795
2956
  }
1796
2957
  finally {
1797
- await this.destroyProbe(app, machineId);
2958
+ await this.destroyProbe(app, machineId, ownsProbeApp);
1798
2959
  }
1799
2960
  }
1800
2961
  /**
@@ -1855,15 +3016,17 @@ class FlyioSandboxProvider {
1855
3016
  return machine.id;
1856
3017
  }
1857
3018
  /**
1858
- * Removes the probe Machine and, in per-project mode, its throwaway app.
3019
+ * Removes the accepted probe Machine and, in per-project mode, only an app
3020
+ * whose create POST this probe itself had accepted. Adopted/unknown apps KEEP.
1859
3021
  *
1860
3022
  * Best-effort by design: a cleanup failure must not turn a completed
1861
3023
  * observation into an error, and `auto_destroy` plus the bounded init sleep
1862
3024
  * already reap the Machine on their own.
1863
3025
  * @param app - The probe app, or `null` when it was never resolved.
1864
- * @param machineId - The probe Machine id, or `null` when it was never created.
3026
+ * @param machineId - The accepted probe Machine id, or `null` when none was created.
3027
+ * @param appCreated - Accepted app-create provenance for this exact probe.
1865
3028
  */
1866
- async destroyProbe(app, machineId) {
3029
+ async destroyProbe(app, machineId, appCreated) {
1867
3030
  if (!app)
1868
3031
  return;
1869
3032
  if (machineId) {
@@ -1877,8 +3040,8 @@ class FlyioSandboxProvider {
1877
3040
  logger.warn('Failed to remove the Fly egress probe Machine', { app, machineId, error });
1878
3041
  }
1879
3042
  }
1880
- // Never in shared-app mode: that app holds real sandboxes.
1881
- if (!this.appPerProject())
3043
+ // Never in shared-app mode, nor for a colliding/adopted/unknown app.
3044
+ if (!this.appPerProject() || !appCreated)
1882
3045
  return;
1883
3046
  try {
1884
3047
  await this.client.request(`/apps/${app}`, { method: 'DELETE', nullOn: [404] });
@@ -1940,12 +3103,13 @@ class FlyioSandboxProvider {
1940
3103
  * @param machineId - The Fly Machine id.
1941
3104
  * @returns A transport callback for {@link execCommand}.
1942
3105
  */
1943
- rawExec(app, machineId) {
3106
+ rawExec(app, machineId, owner) {
1944
3107
  return async (command, timeoutSeconds) => {
1945
3108
  const deadline = Date.now() + EXEC_PRECONDITION_BUDGET_MS;
1946
3109
  for (let attempt = 1;; attempt++) {
1947
3110
  try {
1948
- const response = await this.client.request(`/apps/${app}/machines/${machineId}/exec`, {
3111
+ const path = `/apps/${app}/machines/${machineId}/exec`;
3112
+ const request = {
1949
3113
  method: 'POST',
1950
3114
  body: { command, timeout: timeoutSeconds },
1951
3115
  // The API's own timeout bounds the command; allow the HTTP request a
@@ -1955,7 +3119,10 @@ class FlyioSandboxProvider {
1955
3119
  // exec burst importFiles issues) is Fly API inconsistency under load,
1956
3120
  // not a real absence — retry it alongside the default 429/5xx.
1957
3121
  retryStatuses: [404],
1958
- });
3122
+ };
3123
+ const response = owner
3124
+ ? await this.requestForAttempt(owner, path, request)
3125
+ : await this.client.request(path, request);
1959
3126
  return response ?? {};
1960
3127
  }
1961
3128
  catch (error) {
@@ -2101,7 +3268,7 @@ class FlyioSandboxProvider {
2101
3268
  * @param machineId - The Fly Machine id.
2102
3269
  * @throws {Error} When the Machine has not started within the budget.
2103
3270
  */
2104
- async waitForStarted(app, machineId) {
3271
+ async waitForStarted(app, machineId, owner) {
2105
3272
  const totalSeconds = Math.max(1, this.config.startTimeoutSeconds ?? DEFAULT_START_TIMEOUT_SECONDS);
2106
3273
  const deadline = Date.now() + totalSeconds * 1000;
2107
3274
  let lastState;
@@ -2111,24 +3278,45 @@ class FlyioSandboxProvider {
2111
3278
  // enough: a host that does not have the image cached (every first boot
2112
3279
  // after an image push) pulls it first, which alone can exceed 60 s.
2113
3280
  for (;;) {
3281
+ if (owner) {
3282
+ await this.config.createAttemptJournal?.assertGeneration(owner.attemptId, owner.generation);
3283
+ if (owner.signal.aborted)
3284
+ throw owner.signal.reason;
3285
+ }
2114
3286
  const remaining = Math.ceil((deadline - Date.now()) / 1000);
2115
3287
  const seconds = Math.max(1, Math.min(remaining, MAX_WAIT_SECONDS));
2116
3288
  try {
2117
3289
  await this.client.request(`/apps/${app}/machines/${machineId}/wait?state=started&timeout=${seconds}`,
2118
3290
  // One attempt: a retry would re-block for the whole round, and the
2119
3291
  // state re-read below decides whether another round is warranted.
2120
- { attempts: 1, timeoutMs: (seconds + 15) * 1000 });
3292
+ { attempts: 1, timeoutMs: (seconds + 15) * 1000, signal: owner?.signal });
3293
+ if (owner) {
3294
+ await this.config.createAttemptJournal?.assertGeneration(owner.attemptId, owner.generation);
3295
+ if (owner.signal.aborted)
3296
+ throw owner.signal.reason;
3297
+ }
2121
3298
  return;
2122
3299
  }
2123
3300
  catch (error) {
3301
+ if (owner?.signal.aborted)
3302
+ throw owner.signal.reason;
2124
3303
  lastError = error;
2125
3304
  // The wait endpoint returning an error is not itself proof the Machine
2126
3305
  // failed — re-read the authoritative state before deciding.
2127
3306
  const machine = await this.client
2128
- .request(`/apps/${app}/machines/${machineId}`, { nullOn: [404] })
3307
+ .request(`/apps/${app}/machines/${machineId}`, {
3308
+ nullOn: [404],
3309
+ signal: owner?.signal,
3310
+ })
2129
3311
  .catch(() => null);
2130
- if (machine?.state === 'started')
3312
+ if (machine?.state === 'started') {
3313
+ if (owner) {
3314
+ await this.config.createAttemptJournal?.assertGeneration(owner.attemptId, owner.generation);
3315
+ if (owner.signal.aborted)
3316
+ throw owner.signal.reason;
3317
+ }
2131
3318
  return;
3319
+ }
2132
3320
  lastState = machine?.state;
2133
3321
  // Another round is useful only while the Machine is still on its way
2134
3322
  // up (`created` = provisioning/image pull, `starting` = boot). Any