@molecule/api-code-sandbox-flyio 1.3.6 → 1.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/provider.js CHANGED
@@ -6,16 +6,16 @@
6
6
  *
7
7
  * @module
8
8
  */
9
- import { createHash } from 'node:crypto';
9
+ import { createHash, randomUUID } from 'node:crypto';
10
10
  import { promisify } from 'node:util';
11
11
  import { gzip } from 'node:zlib';
12
12
  import { getLogger } from '@molecule/api-bond';
13
13
  import { t } from '@molecule/api-i18n';
14
- import { FlyApiClient, FlyApiError, normalizeApiUrl } from './api.js';
14
+ import { FlyApiClient, FlyApiError, normalizeApiUrl, retryDelayMs, } from './api.js';
15
15
  import { buildEgressProbeCommand, DEFAULT_EGRESS_POLICY_NAME, DEFAULT_EGRESS_PROBE_TARGETS, describePublicReach, extractPolicyId, extractPolicyPorts, parseEgressAllowedPorts, parseEgressProbeTargets, verdictForProbeExit, } from './egress.js';
16
16
  import { execCommand, toExecResult } from './exec.js';
17
17
  import { assertPrivateRoutesForEnv, decodePrivateRoutes, DEFAULT_PRIVATE_IP_TYPE, encodePrivateRoutes, flycastHost, mergeEgressPorts, parsePrivateServices, PRIVATE_ROUTES_METADATA_SUFFIX, } from './flycast.js';
18
- import { appNameForProject, mapMachineState, parseSandboxId, toSandboxId } from './ids.js';
18
+ import { appNameForProject, appNameNamespace, mapMachineState, parseSandboxId, toSandboxId, } from './ids.js';
19
19
  import { createTemplateStore, MAX_PRESIGN_EXPIRY_SECONDS } from './storage.js';
20
20
  import * as templates from './templates.js';
21
21
  import { shellQuote } from './utilities.js';
@@ -102,6 +102,8 @@ const DEFAULT_PREVIEW_PORT = 5173;
102
102
  const WORKSPACE_PATH = '/workspace';
103
103
  /** Default volume size in GB when `SandboxConfig.resources.diskMB` is unset. */
104
104
  const DEFAULT_VOLUME_GB = 10;
105
+ /** Exact-identity reads allowed after an ambiguous create before its outcome is declared unknown. */
106
+ const CREATE_RECONCILE_ATTEMPTS = 3;
105
107
  /** Page size for the org-wide Machine listing. */
106
108
  const LIST_PAGE_SIZE = 200;
107
109
  /** Hard cap on pages fetched while listing, so a bad cursor cannot loop forever. */
@@ -204,13 +206,13 @@ const MACHINE_CREATE_ATTEMPTS = 6;
204
206
  * means no Machine was created — unlike a 5xx/timeout, where one might have
205
207
  * been. 412 is safe for the same reason: "volume already claimed by machine
206
208
  * <id>" refuses the create. It is what the create answers in the seconds after
207
- * {@link FlyioSandboxProvider.destroyStaleMachines} removed the Machine that
208
- * held the volume — Fly releases the claim asynchronously, and an immediate
209
- * create lost a production deploy to it (2026-10-14; the identical retry a
209
+ * an explicit destroy released the Machine that held the volume — Fly releases
210
+ * the claim asynchronously, and an immediate create lost a production deploy
211
+ * to it (2026-10-14; the identical retry a
210
212
  * minute later succeeded). Only these two are added on top of the client's
211
213
  * default retryable set; neither retry can double-create.
212
214
  */
213
- const MACHINE_CREATE_RETRY_STATUSES = [404, 412];
215
+ const MACHINE_CREATE_RETRY_STATUSES = [400, 404, 412];
214
216
  /**
215
217
  * What Fly answers when it reached the exec endpoint but the connection to the
216
218
  * Machine's exec agent died before the command ran: `412 {"error":
@@ -369,8 +371,11 @@ class FlyioSandboxProvider {
369
371
  * Resolves the Fly region for Machines and volumes.
370
372
  * @returns The region code.
371
373
  */
372
- region() {
373
- return this.config.region ?? process.env.FLY_REGION ?? DEFAULT_REGION;
374
+ region(createRegion) {
375
+ // FLY_REGION is the API Machine's own location on Fly, not a tenant
376
+ // placement decision. Only provider configuration or this create's recorded
377
+ // region may choose where new project resources live.
378
+ return createRegion ?? this.config.region ?? DEFAULT_REGION;
374
379
  }
375
380
  /**
376
381
  * Whether each project gets its own Fly app on its own custom 6PN network.
@@ -480,10 +485,51 @@ class FlyioSandboxProvider {
480
485
  * `<app>.internal` already resolves and Flycast would add nothing.
481
486
  * @returns The declared services, or `undefined` when none are configured.
482
487
  */
483
- privateServices() {
488
+ privateServices(createRegion) {
484
489
  if (!this.appPerProject())
485
490
  return undefined;
486
- return (this.config.privateServices ?? parsePrivateServices(process.env.FLY_SANDBOX_PRIVATE_SERVICES));
491
+ const fallback = this.config.privateServices ?? parsePrivateServices(process.env.FLY_SANDBOX_PRIVATE_SERVICES);
492
+ if (!createRegion || createRegion === DEFAULT_REGION)
493
+ return fallback;
494
+ const suffix = createRegion.replace(/-/g, '_').toUpperCase();
495
+ const variable = `FLY_SANDBOX_PRIVATE_SERVICES_${suffix}`;
496
+ const regional = this.config.privateServicesByRegion?.[createRegion] ??
497
+ parsePrivateServices(process.env[variable]);
498
+ if (!regional && fallback?.length) {
499
+ throw new Error(`No Fly private services are configured for create region "${createRegion}"; ` +
500
+ `set ${variable} (or config.privateServicesByRegion.${createRegion}). ` +
501
+ "Refusing to reuse another region's tenant route.");
502
+ }
503
+ return regional ?? fallback;
504
+ }
505
+ /** Fence and cancel every provisioning mutation at its owning attempt. */
506
+ async requestForAttempt(attempt, path, options) {
507
+ if (attempt.signal.aborted)
508
+ throw attempt.signal.reason;
509
+ await this.config.createAttemptJournal?.assertGeneration(attempt.attemptId, attempt.generation);
510
+ if (attempt.signal.aborted)
511
+ throw attempt.signal.reason;
512
+ const result = await this.client.request(path, { ...options, signal: attempt.signal });
513
+ // A response can arrive after ownership was lost. The write may remain
514
+ // unresolved, but a superseded worker must not journal/adopt its outcome.
515
+ if (attempt.signal.aborted)
516
+ throw attempt.signal.reason;
517
+ await this.config.createAttemptJournal?.assertGeneration(attempt.attemptId, attempt.generation);
518
+ return result;
519
+ }
520
+ /** Authorize a DELETE from the committed retired row on its live lock session. */
521
+ async deleteRetired(intent, generation, path, signal) {
522
+ const journal = this.config.createAttemptJournal;
523
+ if (!journal)
524
+ throw new Error('Retired deletion requires its durable create-attempt journal');
525
+ await journal.authorizeDestruction(intent.attemptId, generation);
526
+ return this.requestForDestruction(path, { method: 'DELETE', nullOn: [404] }, signal);
527
+ }
528
+ /** The sole request dispatch for retirement and settlement, always lock-fenced. */
529
+ async requestForDestruction(path, options, signal) {
530
+ if (signal.aborted)
531
+ throw signal.reason;
532
+ return this.client.request(path, { ...options, signal });
487
533
  }
488
534
  /**
489
535
  * Idempotently ensures the Fly app exists, creating it on its own custom 6PN
@@ -505,27 +551,52 @@ class FlyioSandboxProvider {
505
551
  * @returns `routes`: target app name → allocated private address, for
506
552
  * recording in the Machine's metadata (empty when no private services are
507
553
  * declared); `allocated`: the subset of `routes` this call created, which
508
- * the caller releases when the boot fails and the app is NOT discarded
509
- * (an app that pre-existed is kept, and with it the addresses it already
510
- * held — only this call's own allocations are safe to take back).
554
+ * the caller releases when the boot fails. The app itself always survives:
555
+ * another create can adopt it by name while this call is still failing.
511
556
  */
512
557
  async ensureApp(app, options = {}) {
513
558
  const existing = await this.client.request(`/apps/${app}`, { nullOn: [404] });
514
559
  let createdHere = false;
560
+ if (existing && options.requireCreated) {
561
+ return { routes: {}, allocated: {}, created: false };
562
+ }
515
563
  if (!existing) {
516
564
  const body = { name: app, org_slug: this.orgSlug() };
517
565
  const network = this.networkFor(app);
518
566
  if (network)
519
567
  body.network = network;
520
568
  let created = true;
569
+ const attempt = options.attempt;
570
+ const journal = this.config.createAttemptJournal;
571
+ if (attempt && journal) {
572
+ // Persist who is attempting the create BEFORE Fly can accept it. The
573
+ // separate accepted bit below prevents a pending/unknown POST from
574
+ // becoming deletion authority after a crash or reconciliation error.
575
+ await journal.recordAppProvenance(attempt.attemptId, attempt.generation, 'pending');
576
+ }
521
577
  try {
522
- await this.client.request(`/apps`, { method: 'POST', body });
578
+ if (attempt) {
579
+ await this.requestForAttempt(attempt, `/apps`, { method: 'POST', body });
580
+ }
581
+ else {
582
+ await this.client.request(`/apps`, { method: 'POST', body });
583
+ }
584
+ // Accepted POST provenance is established before later policy/route
585
+ // steps can fail, so diagnostic cleanup can still retire its own app.
586
+ if (options.ownership)
587
+ options.ownership.created = true;
588
+ if (attempt && journal) {
589
+ await journal.recordAppProvenance(attempt.attemptId, attempt.generation, 'created');
590
+ }
523
591
  logger.info('Created Fly sandbox app', { app, network: body.network });
524
592
  }
525
593
  catch (error) {
526
594
  // 409/422 means another concurrent boot created the same app first, which
527
595
  // is exactly the outcome we wanted. Anything else is a real failure.
528
596
  if (error instanceof FlyApiError && (error.status === 409 || error.status === 422)) {
597
+ if (attempt && journal) {
598
+ await journal.recordAppProvenance(attempt.attemptId, attempt.generation, 'adopted');
599
+ }
529
600
  logger.debug('Fly sandbox app already exists', { app });
530
601
  created = false;
531
602
  }
@@ -535,45 +606,18 @@ class FlyioSandboxProvider {
535
606
  }
536
607
  createdHere = created;
537
608
  }
538
- try {
539
- if (createdHere && this.config.assignSharedIpv4)
540
- await this.assignSharedIpv4(app);
541
- await this.applyEgressPolicy(app);
542
- if (options.privateRoutes === false)
543
- return { routes: {}, allocated: {}, created: createdHere };
544
- return {
545
- ...(await this.ensurePrivateRoutes(app, Boolean(existing))),
546
- created: createdHere,
547
- };
548
- }
549
- catch (error) {
550
- if (createdHere)
551
- await this.discardCreatedApp(app, {});
552
- throw error;
609
+ if (options.requireCreated && !createdHere) {
610
+ return { routes: {}, allocated: {}, created: false };
553
611
  }
554
- }
555
- /**
556
- * Deletes an app THIS call created, after a later step of the same create
557
- * failed. Without it every failed create leaves an app behind with its
558
- * addresses and volume: on 2026-09-28, 34 empty `mol-prod-*` apps had piled
559
- * up in production, one per deploy that hit the organization's Machine limit
560
- * (`POST /machines` → 422) after its app was already made. An app that
561
- * existed before the call is never touched — it may hold a working Machine.
562
- * @param app - The app this call created.
563
- * @param privateRoutes - Flycast addresses already allocated for it.
564
- */
565
- async discardCreatedApp(app, privateRoutes) {
566
- try {
567
- await this.client.request(`/apps/${app}`, { method: 'DELETE', nullOn: [404] });
568
- logger.info('Deleted a Fly sandbox app whose create failed', { app });
612
+ if (createdHere && this.config.assignSharedIpv4) {
613
+ await this.assignSharedIpv4(app, options.attempt);
569
614
  }
570
- catch (error) {
571
- logger.error('Failed to delete a Fly sandbox app after its create failed — it leaks', {
572
- app,
573
- error,
574
- });
615
+ await this.applyEgressPolicy(app, options.attempt, options.createRegion);
616
+ if (options.privateRoutes === false) {
617
+ return { routes: {}, allocated: {}, created: createdHere };
575
618
  }
576
- await this.releasePrivateRoutes(privateRoutes);
619
+ const routes = await this.ensurePrivateRoutes(app, Boolean(existing), options.attempt, options.createRegion);
620
+ return { ...routes, created: createdHere };
577
621
  }
578
622
  /**
579
623
  * Allocates a Flycast private address on every declared service app, INTO this
@@ -604,37 +648,42 @@ class FlyioSandboxProvider {
604
648
  * the only addresses safe to release when the boot fails after allocation,
605
649
  * since the recorded ones belong to Machines an earlier boot may still run.
606
650
  */
607
- async ensurePrivateRoutes(app, appExisted) {
608
- const services = this.privateServices();
651
+ async ensurePrivateRoutes(app, appExisted, attempt, createRegion) {
652
+ const services = this.privateServices(createRegion);
609
653
  const network = this.networkFor(app);
610
654
  if (!services?.length || !network)
611
655
  return { routes: {}, allocated: {} };
612
656
  const routes = appExisted ? await this.recordedPrivateRoutes(app) : {};
613
657
  // The addresses THIS call allocated, tracked apart from `routes` so a
614
- // failure part-way down the next list can release exactly them. The caller's
615
- // failure path can only discard the APP (`discardCreatedApp(app, {})`), and
616
- // these addresses live on OTHER apps — so without this, an address already
617
- // allocated for an earlier service stays behind on its target app, inside a
618
- // 6PN whose app is gone, and Fly's listing does not report which network an
619
- // address belongs to: nothing could ever find it there to release it.
658
+ // failure part-way down the next list can release exactly them. These
659
+ // addresses live on OTHER apps, so deleting this project's app would not
660
+ // release them; Fly's listing also cannot reconstruct their target network.
620
661
  const allocated = {};
621
- try {
622
- for (const service of services) {
623
- if (routes[service.app])
624
- continue;
625
- const ip = await this.allocatePrivateAddress(service, network);
626
- if (ip) {
627
- routes[service.app] = ip;
628
- allocated[service.app] = ip;
629
- }
662
+ for (const service of services) {
663
+ if (routes[service.app])
664
+ continue;
665
+ if (attempt && this.config.createAttemptJournal) {
666
+ await this.config.createAttemptJournal.recordAddress(attempt.attemptId, {
667
+ targetApp: service.app,
668
+ network,
669
+ owned: false,
670
+ resolved: false,
671
+ });
672
+ }
673
+ const ip = await this.allocatePrivateAddress(service, network, attempt);
674
+ if (attempt && this.config.createAttemptJournal) {
675
+ await this.config.createAttemptJournal.recordAddress(attempt.attemptId, {
676
+ targetApp: service.app,
677
+ network,
678
+ ...(ip ? { ip } : {}),
679
+ owned: Boolean(ip),
680
+ resolved: true,
681
+ });
682
+ }
683
+ if (ip) {
684
+ routes[service.app] = ip;
685
+ allocated[service.app] = ip;
630
686
  }
631
- }
632
- catch (error) {
633
- // Only what this call allocated. An address recorded by an EARLIER boot
634
- // (`appExisted`) is left alone: the app survived this failure and may
635
- // still hold a Machine reaching through that route.
636
- await this.releasePrivateRoutes(allocated);
637
- throw error;
638
687
  }
639
688
  return { routes, allocated };
640
689
  }
@@ -667,50 +716,26 @@ class FlyioSandboxProvider {
667
716
  return {};
668
717
  }
669
718
  /**
670
- * Destroys any Machine already in this project's app before a fresh create.
671
- *
672
- * A create that fails after the Machine exists — a mid-scaffold error, a
673
- * retried queue job — leaves a Machine holding the app's single-attach volume.
674
- * The next create's `POST /machines` then fails with `412 volume already
675
- * claimed by machine <id>`, and the project can never recover on its own. The
676
- * app is one-per-project and holds exactly one sandbox, so any managed Machine
677
- * still here is a stale partial: destroy it (force covers running/suspended) to
678
- * free the volume and make create idempotent per project.
679
- * @param app - The Fly app name.
719
+ * Read and normalize Fly's assignment list once for allocation and settlement.
720
+ * Current Fly returns `{ips: [...]}` and a network object; the older array /
721
+ * `ip_assignments` shapes remain readable for compatibility with saved clients.
680
722
  */
681
- async destroyStaleMachines(app) {
682
- const managedKey = `${this.metadataPrefix}.managed`;
683
- let machines;
684
- try {
685
- machines = await this.client.request(`/apps/${app}/machines`, {
686
- nullOn: [404],
687
- });
688
- }
689
- catch (error) {
690
- // A listing we cannot read is not proof the app is empty. A hidden Machine
691
- // would fail the create with 412 regardless, so surface it and fall
692
- // through rather than pretend the app is clean.
693
- logger.warn('Could not list Machines before create; a stale Machine may still hold the volume', { app, error });
694
- return;
695
- }
696
- for (const machine of Array.isArray(machines) ? machines : []) {
697
- if (machine.config?.metadata?.[managedKey] !== 'true')
698
- continue;
699
- try {
700
- await this.client.request(`/apps/${app}/machines/${machine.id}?force=true`, {
701
- method: 'DELETE',
702
- nullOn: [404],
703
- });
704
- logger.info('Destroyed a stale Machine before re-creating the sandbox', {
705
- app,
706
- machineId: machine.id,
707
- });
708
- }
709
- catch (error) {
710
- logger.warn('Failed to destroy a stale Machine before create; create may fail with ' +
711
- '412 volume already claimed', { app, machineId: machine.id, error });
723
+ async listPrivateAssignments(app, signal) {
724
+ const listed = signal
725
+ ? await this.requestForDestruction(`/apps/${app}/ip_assignments`, {}, signal)
726
+ : await this.client.request(`/apps/${app}/ip_assignments`);
727
+ const assignments = Array.isArray(listed) ? listed : (listed?.ips ?? listed?.ip_assignments);
728
+ if (!assignments)
729
+ return null;
730
+ return assignments.map((assignment) => {
731
+ if (typeof assignment.network === 'string')
732
+ return assignment;
733
+ const network = assignment.network;
734
+ if (!network || network.org_slug !== this.orgSlug()) {
735
+ return { ...assignment, network: undefined };
712
736
  }
713
- }
737
+ return { ...assignment, network: network.name ?? '' };
738
+ });
714
739
  }
715
740
  /**
716
741
  * Allocates ONE Flycast private address on a target app, for this project's
@@ -728,7 +753,7 @@ class FlyioSandboxProvider {
728
753
  * address — an allocation this provider can neither verify nor release later
729
754
  * is untracked state, and the boot is the last moment it can be reported.
730
755
  */
731
- async allocatePrivateAddress(service, network) {
756
+ async allocatePrivateAddress(service, network, attempt) {
732
757
  let assignment;
733
758
  try {
734
759
  // NO org_slug here. The ip_assignments endpoint derives the org from the
@@ -737,25 +762,28 @@ class FlyioSandboxProvider {
737
762
  // slug that app creation REQUIRES is refused here; omitting it returns 200
738
763
  // and allocates the address). app creation still sends org_slug — the two
739
764
  // endpoints genuinely differ.
740
- assignment = await this.client.request(`/apps/${service.app}/ip_assignments`, {
765
+ const request = {
741
766
  method: 'POST',
742
767
  body: {
743
768
  type: this.config.privateIpAssignmentType ?? DEFAULT_PRIVATE_IP_TYPE,
744
769
  network,
745
770
  },
746
- });
771
+ };
772
+ assignment = attempt
773
+ ? await this.requestForAttempt(attempt, `/apps/${service.app}/ip_assignments`, request)
774
+ : await this.client.request(`/apps/${service.app}/ip_assignments`, request);
747
775
  }
748
776
  catch (error) {
749
- // A conflict means an address for this (app, network) pair is already
750
- // there — from a boot that crashed between allocating and recording. The
751
- // ROUTE is what the sandbox needs and it exists, so this is not a boot
752
- // failure; it is an address this provider can no longer release, because
753
- // Fly's listing does not report which network an address belongs to.
777
+ // A conflict can be the duplicate response to an accepted write whose
778
+ // first response was lost. Resolve it by target app AND network, then the
779
+ // caller journals the concrete IP for adoption or later release.
754
780
  if (error instanceof FlyApiError && (error.status === 409 || error.status === 422)) {
755
- logger.warn('A Fly private address for this project’s network already exists on the target app — ' +
756
- 'the route works, but this one cannot be released on destroy. Reconcile with ' +
757
- '`fly ips list --app <target>`.', { target: service.app, network, status: error.status });
758
- return undefined;
781
+ const assignments = await this.listPrivateAssignments(service.app, attempt?.signal);
782
+ const existing = assignments?.find((candidate) => candidate.network === network);
783
+ if (existing?.ip)
784
+ return existing.ip;
785
+ throw new Error(`Fly rejected a private-address allocation for "${service.app}" on "${network}", ` +
786
+ 'but no matching assignment could be resolved', { cause: error });
759
787
  }
760
788
  throw new Error(t('codeSandbox.flyio.error.privateRouteFailed', { app: service.app, network }, {
761
789
  defaultValue: `Could not allocate a Flycast private address on Fly app "${service.app}" for ` +
@@ -781,28 +809,42 @@ class FlyioSandboxProvider {
781
809
  });
782
810
  return assignment.ip;
783
811
  }
812
+ /** Validate the exact IP's current network; metadata alone is not DELETE authority. */
813
+ async privateRouteMatchesNetwork(target, ip, network, signal) {
814
+ if (!network)
815
+ return false;
816
+ const assignments = await this.listPrivateAssignments(target, signal);
817
+ return Boolean(assignments?.some((candidate) => candidate.ip === ip && candidate.network === network));
818
+ }
784
819
  /**
785
- * Releases the Flycast addresses a project's sandbox recorded.
786
- *
787
- * Best-effort: the sandbox is being destroyed either way, and an address left
788
- * behind is a reconciliation item (`fly ips list --app <target>`) rather than
789
- * a failed teardown. It routes into a 6PN that no longer has any app in it.
790
- * @param routes - Target app name → private address, from Machine metadata.
820
+ * Releases only Flycast addresses independently validated for this network.
821
+ * Metadata identifies candidates; it never authorizes a cross-network DELETE.
791
822
  */
792
- async releasePrivateRoutes(routes) {
823
+ async releasePrivateRoutes(routes, signal, network) {
824
+ let released = true;
793
825
  for (const [target, ip] of Object.entries(routes)) {
794
826
  try {
795
- await this.client.request(`/apps/${target}/ip_assignments/${encodeURIComponent(ip)}`, {
796
- method: 'DELETE',
797
- nullOn: [404],
798
- });
827
+ if (!(await this.privateRouteMatchesNetwork(target, ip, network, signal))) {
828
+ released = false;
829
+ logger.warn('Kept a Fly private (Flycast) address with unvalidated network ownership', {
830
+ target,
831
+ ip,
832
+ network,
833
+ });
834
+ continue;
835
+ }
836
+ await this.requestForDestruction(`/apps/${target}/ip_assignments/${encodeURIComponent(ip)}`, { method: 'DELETE', nullOn: [404] }, signal);
799
837
  logger.debug('Released a Fly private (Flycast) address', { target, ip });
800
838
  }
801
839
  catch (error) {
840
+ if (signal.aborted)
841
+ throw signal.reason;
842
+ released = false;
802
843
  logger.warn('Failed to release a Fly private (Flycast) address — it will keep pointing at a 6PN ' +
803
844
  'with no apps in it until it is reconciled with `fly ips list`', { target, ip, error });
804
845
  }
805
846
  }
847
+ return released;
806
848
  }
807
849
  /**
808
850
  * Resolves the ports the egress network policy allows, or `undefined` when no
@@ -821,8 +863,8 @@ class FlyioSandboxProvider {
821
863
  * about a rule with no ports, so accepting `[]` would ship a security
822
864
  * control whose behaviour is a guess.
823
865
  */
824
- egressAllowedPorts() {
825
- return mergeEgressPorts(this.configuredEgressPorts(), this.privateServices());
866
+ egressAllowedPorts(createRegion) {
867
+ return mergeEgressPorts(this.configuredEgressPorts(), this.privateServices(createRegion));
826
868
  }
827
869
  /**
828
870
  * Resolves ONLY the ports the operator configured, before the declared private
@@ -855,8 +897,8 @@ class FlyioSandboxProvider {
855
897
  * @param app - The Fly app name.
856
898
  * @returns Nothing; resolves once the policy is in place (or none was configured).
857
899
  */
858
- async applyEgressPolicy(app) {
859
- const ports = this.egressAllowedPorts();
900
+ async applyEgressPolicy(app, attempt, createRegion) {
901
+ const ports = this.egressAllowedPorts(createRegion);
860
902
  if (!ports)
861
903
  return;
862
904
  const name = this.config.egressPolicyName ?? DEFAULT_EGRESS_POLICY_NAME;
@@ -888,7 +930,18 @@ class FlyioSandboxProvider {
888
930
  selector: { all: true },
889
931
  rules: [{ action: 'allow', direction: 'egress', ports }],
890
932
  };
891
- await this.client.request(`/apps/${app}/network_policies`, { method: 'POST', body: policy });
933
+ if (attempt) {
934
+ await this.requestForAttempt(attempt, `/apps/${app}/network_policies`, {
935
+ method: 'POST',
936
+ body: policy,
937
+ });
938
+ }
939
+ else {
940
+ await this.client.request(`/apps/${app}/network_policies`, {
941
+ method: 'POST',
942
+ body: policy,
943
+ });
944
+ }
892
945
  logger.info('Applied Fly egress network policy', {
893
946
  app,
894
947
  name,
@@ -966,7 +1019,7 @@ class FlyioSandboxProvider {
966
1019
  * rather than failing the boot.
967
1020
  * @param app - The Fly app name.
968
1021
  */
969
- async assignSharedIpv4(app) {
1022
+ async assignSharedIpv4(app, attempt) {
970
1023
  try {
971
1024
  // ONLY `type` — verified against the live API 2026-08-09. Both `region`
972
1025
  // ("region selection is not supported for this IP type", 400) and
@@ -974,37 +1027,51 @@ class FlyioSandboxProvider {
974
1027
  // sandbox app WITHOUT a public IP, so its preview URL never routed and the
975
1028
  // IDE hung on "Loading preview…" forever. The org derives from the app and
976
1029
  // a shared v4 is global (no region).
977
- await this.client.request(`/apps/${app}/ip_assignments`, {
1030
+ const request = {
978
1031
  method: 'POST',
979
- body: {
980
- type: this.config.ipAssignmentType ?? 'shared_v4',
981
- },
982
- });
1032
+ body: { type: this.config.ipAssignmentType ?? 'shared_v4' },
1033
+ };
1034
+ if (attempt)
1035
+ await this.requestForAttempt(attempt, `/apps/${app}/ip_assignments`, request);
1036
+ else
1037
+ await this.client.request(`/apps/${app}/ip_assignments`, request);
983
1038
  }
984
1039
  catch (error) {
1040
+ if (attempt?.signal.aborted)
1041
+ throw attempt.signal.reason;
1042
+ if (attempt) {
1043
+ await this.config.createAttemptJournal?.assertGeneration(attempt.attemptId, attempt.generation);
1044
+ }
985
1045
  logger.warn('Failed to assign a shared IPv4 to the Fly sandbox app — the public preview URL will ' +
986
1046
  'not resolve until an address is allocated', { app, error });
987
1047
  }
988
1048
  }
989
1049
  /**
990
- * Sanitizes a caller-supplied volume name into a valid Fly volume name.
991
- *
992
- * Fly requires `[a-z0-9_]` (lowercase only) and at most 30 characters — it
993
- * rejects anything else with `400 "name only allows lowercase alphanumeric
994
- * characters and underscores with at most 30 characters"`. Callers pass names
995
- * like `mol-<uuid>` (40 chars, hyphens): lowercase, map every other character
996
- * to `_`, and when the result is too long, keep a readable prefix plus a
997
- * deterministic hash suffix so the name stays stable (`findVolume` re-derives
998
- * the same name on the idempotent path) and unique.
999
- * @param name - The caller's volume name.
1000
- * @returns A sanitized Fly volume name, `[a-z0-9_]` and <= 30 chars.
1050
+ * Return the readable, sanitized identity shared by attempts for one caller label.
1051
+ * @param name - The caller's volume label.
1052
+ * @returns The `[a-z0-9_]` stem used by {@link volumeName}.
1053
+ */
1054
+ volumeNameStem(name) {
1055
+ const sanitized = name.toLowerCase().replace(/[^a-z0-9_]/g, '_');
1056
+ return `${sanitized.slice(0, 17)}_`;
1057
+ }
1058
+ /**
1059
+ * Give one create attempt a unique Fly volume name (Fly permits duplicate names).
1060
+ * @param name - The caller's volume label.
1061
+ * @param attemptId - The Machine create attempt's identity.
1062
+ * @returns A sanitized name of at most 30 characters, including the full attempt's digest.
1001
1063
  */
1002
- volumeName(name) {
1064
+ volumeName(name, attemptId) {
1065
+ const digest = createHash('sha256').update(`${name}\0${attemptId}`).digest('hex').slice(0, 12);
1066
+ return `${this.volumeNameStem(name)}${digest}`;
1067
+ }
1068
+ /** The stable pre-attempt name under which durable workspaces were created. */
1069
+ workspaceVolumeName(name) {
1003
1070
  const sanitized = name.toLowerCase().replace(/[^a-z0-9_]/g, '_');
1004
1071
  if (sanitized.length <= 30)
1005
1072
  return sanitized;
1006
- const hash = createHash('sha256').update(name).digest('hex').slice(0, 12);
1007
- return `${sanitized.slice(0, 17)}_${hash}`;
1073
+ const digest = createHash('sha256').update(name).digest('hex').slice(0, 12);
1074
+ return `${sanitized.slice(0, 17)}_${digest}`;
1008
1075
  }
1009
1076
  /**
1010
1077
  * Idempotently ensures a volume exists in the app and returns its id.
@@ -1014,28 +1081,138 @@ class FlyioSandboxProvider {
1014
1081
  * size are all known — rather than through the core's optional
1015
1082
  * `createVolume(name)`; see the module `@remarks`.
1016
1083
  * @param app - The Fly app name.
1017
- * @param name - The caller's volume name.
1084
+ * @param flyName - This create attempt's unique, already-sanitized volume name.
1018
1085
  * @param sizeGB - Size to create the volume with, in whole GB.
1019
- * @returns The Fly volume id.
1086
+ * @param compute - The exact guest sent with the Machine create.
1087
+ * @returns The Fly volume id and whether this call created the volume.
1020
1088
  */
1021
- async ensureVolume(app, name, sizeGB) {
1022
- const flyName = this.volumeName(name);
1023
- const existing = await this.findVolume(app, flyName);
1024
- if (existing)
1025
- return existing.id;
1026
- const created = await this.client.request(`/apps/${app}/volumes`, {
1089
+ async ensureVolume(app, projectId, name, attempt, sizeGB, compute, region) {
1090
+ const journal = this.config.createAttemptJournal;
1091
+ const bound = await journal?.findWorkspace(projectId, name);
1092
+ if (bound?.id) {
1093
+ const volumes = await this.client.request(`/apps/${app}/volumes`, {
1094
+ signal: attempt.signal,
1095
+ });
1096
+ const existing = (volumes ?? []).find((volume) => volume.id === bound.id && volume.state !== 'destroyed' && volume.state !== 'destroying');
1097
+ if (!existing) {
1098
+ throw new Error(`Durable workspace volume "${bound.id}" cannot be read; refusing to replace it`);
1099
+ }
1100
+ await journal?.recordVolume(attempt.attemptId, {
1101
+ ...bound,
1102
+ id: existing.id,
1103
+ logicalName: name,
1104
+ workspace: true,
1105
+ resolved: true,
1106
+ createdByAttemptId: undefined,
1107
+ adoptedByAttemptId: attempt.attemptId,
1108
+ });
1109
+ return { id: existing.id, name: bound.name, workspace: true };
1110
+ }
1111
+ // Pre-attempt releases used a deterministic name. Adopt that legacy volume
1112
+ // by ID once; all later recreation uses the durable binding above.
1113
+ const durableName = this.workspaceVolumeName(name);
1114
+ const existing = await this.findVolume(app, durableName, attempt.signal);
1115
+ if (existing) {
1116
+ await journal?.recordVolume(attempt.attemptId, {
1117
+ id: existing.id,
1118
+ name: durableName,
1119
+ logicalName: name,
1120
+ workspace: true,
1121
+ resolved: true,
1122
+ adoptedByAttemptId: attempt.attemptId,
1123
+ });
1124
+ return { id: existing.id, name: durableName, workspace: true };
1125
+ }
1126
+ const attemptName = this.volumeName(name, attempt.attemptId);
1127
+ await journal?.recordVolume(attempt.attemptId, {
1128
+ name: attemptName,
1129
+ logicalName: name,
1130
+ workspace: true,
1131
+ resolved: false,
1132
+ createdByAttemptId: attempt.attemptId,
1133
+ });
1134
+ try {
1135
+ const created = await this.createFreshVolume(app, attemptName, attempt, sizeGB, compute, region);
1136
+ await journal?.recordVolume(attempt.attemptId, {
1137
+ id: created.id,
1138
+ name: attemptName,
1139
+ logicalName: name,
1140
+ workspace: false,
1141
+ resolved: true,
1142
+ createdByAttemptId: created.accepted ? attempt.attemptId : undefined,
1143
+ adoptedByAttemptId: created.accepted ? undefined : attempt.attemptId,
1144
+ });
1145
+ return { id: created.id, name: attemptName, workspace: false };
1146
+ }
1147
+ catch (error) {
1148
+ if (error instanceof FlyApiError &&
1149
+ error.method.toUpperCase() === 'POST' &&
1150
+ error.path === `/apps/${app}/volumes` &&
1151
+ error.status >= 400 &&
1152
+ error.status < 500 &&
1153
+ ![409, 422].includes(error.status)) {
1154
+ // A conflict/validation response can describe a write whose earlier
1155
+ // response was lost, so those outcomes stay unresolved. The 429 replay
1156
+ // contract treats rate-limit rejection as definite; other non-conflict
1157
+ // 4xx responses reject this unique attempt name before creating it.
1158
+ await journal?.recordVolume(attempt.attemptId, {
1159
+ name: attemptName,
1160
+ logicalName: name,
1161
+ workspace: false,
1162
+ resolved: true,
1163
+ });
1164
+ }
1165
+ throw error;
1166
+ }
1167
+ }
1168
+ /**
1169
+ * Creates a volume with the expected Machine's guest as its placement hint.
1170
+ * A 429 is safe to replay directly. After a transport failure or 5xx, bounded
1171
+ * lookups reconcile the attempt's unique name; if it stays invisible the create
1172
+ * fails rather than replaying an ambiguous POST that could duplicate the volume.
1173
+ * @param app - The Fly app name.
1174
+ * @param flyName - This create attempt's unique, already-sanitized volume name.
1175
+ * @param sizeGB - Size to create the volume with, in whole GB.
1176
+ * @param compute - The exact guest sent with the Machine create.
1177
+ * @returns The new Fly volume id.
1178
+ */
1179
+ async createFreshVolume(app, flyName, attempt, sizeGB, compute, region) {
1180
+ let adopted = false;
1181
+ const created = await this.requestForAttempt(attempt, `/apps/${app}/volumes`, {
1027
1182
  method: 'POST',
1028
1183
  body: {
1029
1184
  name: flyName,
1030
- region: this.region(),
1185
+ region,
1031
1186
  size_gb: sizeGB,
1032
1187
  encrypted: true,
1188
+ compute,
1189
+ },
1190
+ reconcile: async (error) => {
1191
+ // A 429 is Fly's confirmation that this request was rejected. Every
1192
+ // other retryable outcome is ambiguous: never replay it until a few
1193
+ // name lookups have had a chance to find the volume Fly may have made.
1194
+ if (error.status !== 0 && error.status < 500)
1195
+ return undefined;
1196
+ for (let lookup = 1; lookup <= CREATE_RECONCILE_ATTEMPTS; lookup++) {
1197
+ const existing = await this.findVolume(app, flyName, attempt.signal);
1198
+ if (existing) {
1199
+ adopted = true;
1200
+ return existing;
1201
+ }
1202
+ if (lookup < CREATE_RECONCILE_ATTEMPTS) {
1203
+ await this.client.sleep(retryDelayMs(lookup));
1204
+ }
1205
+ }
1206
+ // Fly permits duplicate volume names. Negative reads therefore cannot
1207
+ // authorize another POST: stop with the outcome unknown and let the
1208
+ // owning app's durable reconciler reclaim anything the write made.
1209
+ throw new Error(`Fly volume create "${flyName}" has an unknown outcome; refusing to replay it`, { cause: error });
1033
1210
  },
1034
1211
  });
1035
1212
  if (!created?.id) {
1036
1213
  throw new Error(t('codeSandbox.flyio.error.volumeCreateFailed', { name: flyName }, { defaultValue: `Fly volume "${flyName}" was created but the API returned no id.` }));
1037
1214
  }
1038
- return created.id;
1215
+ return { id: created.id, accepted: !adopted };
1039
1216
  }
1040
1217
  /**
1041
1218
  * Finds a volume in an app by name, ignoring ones already being destroyed.
@@ -1043,9 +1220,10 @@ class FlyioSandboxProvider {
1043
1220
  * @param flyName - The sanitized Fly volume name.
1044
1221
  * @returns The matching volume, or `null`.
1045
1222
  */
1046
- async findVolume(app, flyName) {
1223
+ async findVolume(app, flyName, signal) {
1047
1224
  const volumes = await this.client.request(`/apps/${app}/volumes`, {
1048
1225
  nullOn: [404],
1226
+ signal,
1049
1227
  });
1050
1228
  if (!Array.isArray(volumes))
1051
1229
  return null;
@@ -1073,7 +1251,21 @@ class FlyioSandboxProvider {
1073
1251
  * @param config - Project id, image, env, volume name, template, labels and resource limits.
1074
1252
  * @returns A `Sandbox` bound to the new Machine.
1075
1253
  */
1076
- async create(config) {
1254
+ /**
1255
+ * Return the collision-free app name for a complete caller/project identity.
1256
+ * @param projectId - The complete project or per-deploy identity.
1257
+ * @returns The DNS-safe, at-most-63-character app name.
1258
+ */
1259
+ appNameForProject(projectId) {
1260
+ return appNameForProject(this.appPrefix(), projectId);
1261
+ }
1262
+ /**
1263
+ * Create, reconcile and boot one uniquely identified Fly Machine attempt.
1264
+ * Automatic cleanup deletes only a volume or exact Machine ID with this attempt's accepted-create provenance. Apps always remain KEEP for deliberate retirement; adopted, inherited, listed, older-writer and ambiguous resources are also KEPT.
1265
+ * @param config - The sandbox image, project, storage and resource configuration.
1266
+ * @returns The running sandbox.
1267
+ */
1268
+ async create(config, signal = new AbortController().signal) {
1077
1269
  // Resolved BEFORE anything is provisioned, so a bad template id costs no app,
1078
1270
  // no volume and no Machine.
1079
1271
  let manifest = null;
@@ -1093,49 +1285,60 @@ class FlyioSandboxProvider {
1093
1285
  // is checked too — the host is unroutable however the value arrives, and
1094
1286
  // that env is exactly where a control plane puts the values it refuses to
1095
1287
  // leave in a Machine config.
1096
- assertPrivateRoutesForEnv({ ...config.env, ...config.selfDeliveredEnv }, this.privateServices());
1288
+ const createRegion = this.region(config.region);
1289
+ assertPrivateRoutesForEnv({ ...config.env, ...config.selfDeliveredEnv }, this.privateServices(createRegion));
1097
1290
  const app = this.resolveApp(config.projectId);
1098
- const { routes: privateRoutes, allocated, created } = await this.ensureApp(app);
1291
+ const createAttemptId = randomUUID();
1292
+ const journal = this.config.createAttemptJournal;
1293
+ // The SQL journal stays closed until pre-journal/provenance-ignoring writers
1294
+ // have drained. No durable row or Fly resource is created while it is closed.
1295
+ await journal?.assertCreateAllowed();
1296
+ await journal?.begin({
1297
+ attemptId: createAttemptId,
1298
+ projectId: config.projectId,
1299
+ ownerProjectId: String(config.labels?.['molecule.production'] ?? config.projectId),
1300
+ appName: app,
1301
+ });
1302
+ const attempt = { attemptId: createAttemptId, generation: 1, signal };
1099
1303
  try {
1100
- return await this.createInApp(app, privateRoutes, config, manifest);
1304
+ const { routes: privateRoutes } = await this.ensureApp(app, {
1305
+ attempt,
1306
+ createRegion,
1307
+ });
1308
+ const sandbox = await this.createInApp(app, privateRoutes, config, manifest, attempt);
1309
+ if (signal.aborted)
1310
+ throw signal.reason;
1311
+ await journal?.assertGeneration(attempt.attemptId, attempt.generation);
1312
+ await journal?.markState(createAttemptId, 'serving', sandbox.id);
1313
+ return sandbox;
1101
1314
  }
1102
1315
  catch (error) {
1103
- if (created && this.appPerProject()) {
1104
- // The whole app goes, addresses included — discardCreatedApp releases
1105
- // every route it was handed (a created app holds only this call's).
1106
- await this.discardCreatedApp(app, privateRoutes);
1107
- }
1108
- else {
1109
- // The app SURVIVES this failure (it pre-existed, or it is the shared
1110
- // app), so the addresses this call allocated would otherwise stay
1111
- // behind on their target apps with nothing recording them: the
1112
- // Machine that would have carried the metadata was never created, and
1113
- // Fly's listing cannot say which network an address belongs to. Take
1114
- // back exactly this call's own allocations — never the routes an
1115
- // earlier boot recorded, whose Machines may still be running.
1116
- await this.releasePrivateRoutes(allocated);
1117
- }
1316
+ // Create has NO deletion path. The durable attempt is settled by the
1317
+ // separate reconciler after this failure (or after its abandonment window).
1318
+ await journal?.markState(createAttemptId, 'failed').catch((journalError) => {
1319
+ logger.warn('Could not mark a Fly create attempt failed', {
1320
+ app,
1321
+ createAttemptId,
1322
+ error: journalError,
1323
+ });
1324
+ });
1118
1325
  throw error;
1119
1326
  }
1120
1327
  }
1121
1328
  /**
1122
- * The part of {@link create} that runs once the app exists: clear stale
1123
- * Machines, provision the volume, create the Machine and wait for it.
1329
+ * The part of {@link create} that provisions the volume, creates the Machine and waits for it.
1124
1330
  * @param app - The app.
1125
1331
  * @param privateRoutes - Flycast addresses allocated for the app.
1126
1332
  * @param config - The sandbox config.
1127
1333
  * @param manifest - The template manifest, when creating from a template.
1128
1334
  * @returns The running sandbox.
1129
1335
  */
1130
- async createInApp(app, privateRoutes, config, manifest) {
1131
- // Clear any stale Machine from a prior failed/partial create before
1132
- // provisioning — otherwise it still holds the app's single-attach volume and
1133
- // `POST /machines` fails with `412 volume already claimed`. Makes create
1134
- // idempotent so a retried queue job (or a user re-clicking Start) recovers.
1135
- await this.destroyStaleMachines(app);
1336
+ async createInApp(app, privateRoutes, config, manifest, attempt) {
1337
+ const createAttemptId = attempt.attemptId;
1136
1338
  const metadata = {
1137
1339
  [`${this.metadataPrefix}.projectId`]: config.projectId,
1138
1340
  [`${this.metadataPrefix}.managed`]: 'true',
1341
+ [`${this.metadataPrefix}.createAttempt`]: createAttemptId,
1139
1342
  // Recorded so `destroy()` can release exactly the addresses this project
1140
1343
  // allocated. Fly's IP listing does not report which network an address
1141
1344
  // belongs to, so nothing else can reconstruct this.
@@ -1181,34 +1384,112 @@ class FlyioSandboxProvider {
1181
1384
  // the image's idle command and serving nothing.
1182
1385
  if (config.command?.length)
1183
1386
  machineConfig.init = { exec: [...config.command] };
1184
- if (config.volumeName) {
1185
- metadata[`${this.metadataPrefix}.volumeName`] = config.volumeName;
1186
- const sizeGB = config.resources?.diskMB
1387
+ const volumeName = config.volumeName;
1388
+ if (volumeName) {
1389
+ metadata[`${this.metadataPrefix}.volumeName`] = volumeName;
1390
+ const volumeSizeGB = config.resources?.diskMB
1187
1391
  ? Math.max(1, Math.ceil(config.resources.diskMB / 1024))
1188
1392
  : (this.config.defaultVolumeGB ?? DEFAULT_VOLUME_GB);
1189
- const volumeId = await this.ensureVolume(app, config.volumeName, sizeGB);
1393
+ const volume = await this.ensureVolume(app, config.projectId, volumeName, attempt, volumeSizeGB, machineConfig.guest ?? {}, this.region(config.region));
1190
1394
  // The caller may scope the volume to a subtree (e.g. /workspace/<appDir>)
1191
1395
  // so regenerable tooling (node_modules) stays on the fast image-backed
1192
1396
  // rootfs and only durable state pays the volume's small-file IO cost.
1193
- machineConfig.mounts = [{ volume: volumeId, path: config.volumeMountPath ?? WORKSPACE_PATH }];
1397
+ machineConfig.mounts = [{ volume: volume.id, path: config.volumeMountPath ?? WORKSPACE_PATH }];
1194
1398
  }
1195
1399
  const service = this.previewService(config.idleAction);
1196
1400
  if (service)
1197
1401
  machineConfig.services = [service];
1198
- const machine = await this.client.request(`/apps/${app}/machines`, {
1199
- method: 'POST',
1200
- body: { region: this.region(), config: machineConfig },
1201
- // A just-created app propagates to the Machines API asynchronously, so this
1202
- // POST can 404 "app not found" for a few seconds. Retry the 404 (safe: no
1203
- // Machine is created when the app is not found) across the client's backoff.
1204
- attempts: MACHINE_CREATE_ATTEMPTS,
1205
- retryStatuses: MACHINE_CREATE_RETRY_STATUSES,
1206
- });
1402
+ // The POST intent must survive a crash before dispatch. A NULL outcome on
1403
+ // an older row is unknown too; only a new not-started/rejected row proves
1404
+ // that no accepted Machine identity can be hidden by a later empty list.
1405
+ await this.config.createAttemptJournal?.recordMachineCreateOutcome(createAttemptId, attempt.generation, 'pending');
1406
+ let machine;
1407
+ let machineCreateResponseLost = false;
1408
+ try {
1409
+ machine = await this.requestForAttempt(attempt, `/apps/${app}/machines`, {
1410
+ method: 'POST',
1411
+ body: {
1412
+ name: createAttemptId,
1413
+ region: this.region(config.region),
1414
+ config: machineConfig,
1415
+ },
1416
+ // A just-created app propagates to the Machines API asynchronously, so this
1417
+ // POST can 404 "app not found" for a few seconds. Retry the 404 (safe: no
1418
+ // Machine is created when the app is not found) across the client's backoff.
1419
+ attempts: MACHINE_CREATE_ATTEMPTS,
1420
+ retryStatuses: MACHINE_CREATE_RETRY_STATUSES,
1421
+ // A 412 is transient only for the specific released-volume-claim lag.
1422
+ // Placement capacity must reach the caller's bounded retry at once,
1423
+ // with the volume preserved; every other 412 remains a real rejection.
1424
+ retryWhen: (error) => {
1425
+ if (error.status === 400)
1426
+ return /volume not found/i.test(error.body);
1427
+ if (error.status !== 412)
1428
+ return true;
1429
+ return (/volume already claimed by machine/i.test(error.body) ||
1430
+ /volume_placement_capacity/i.test(error.body));
1431
+ },
1432
+ reconcile: async (error) => {
1433
+ // Rate limiting and explicit precondition rejections did not create a
1434
+ // Machine, so their normal bounded replay is safe. A transport/5xx
1435
+ // result is ambiguous: reconcile ONLY this attempt's recorded identity
1436
+ // and never issue another create when the reads cannot find it.
1437
+ if (error.status === 429 || (error.status >= 400 && error.status < 500)) {
1438
+ return undefined;
1439
+ }
1440
+ // A Machine recovered by listing after a lost response is adoption
1441
+ // evidence, not an accepted POST response and never delete authority.
1442
+ machineCreateResponseLost = true;
1443
+ await this.config.createAttemptJournal?.recordMachineCreateOutcome(createAttemptId, attempt.generation, 'unknown');
1444
+ for (let lookup = 1; lookup <= CREATE_RECONCILE_ATTEMPTS; lookup++) {
1445
+ const machines = await this.client.request(`/apps/${app}/machines`, {
1446
+ nullOn: [404],
1447
+ signal: attempt.signal,
1448
+ });
1449
+ const existing = (machines ?? []).find((candidate) => candidate.name === createAttemptId ||
1450
+ candidate.config?.metadata?.[`${this.metadataPrefix}.createAttempt`] ===
1451
+ createAttemptId);
1452
+ if (existing) {
1453
+ await this.config.createAttemptJournal?.recordMachineObservation(createAttemptId, attempt.generation, existing.id);
1454
+ return existing;
1455
+ }
1456
+ if (lookup < CREATE_RECONCILE_ATTEMPTS) {
1457
+ await this.client.sleep(retryDelayMs(lookup));
1458
+ }
1459
+ }
1460
+ throw new Error(`Fly Machine create "${createAttemptId}" in app "${app}" has an unknown outcome; refusing to replay it`, { cause: error });
1461
+ },
1462
+ });
1463
+ }
1464
+ catch (error) {
1465
+ if (attempt.signal.aborted)
1466
+ throw attempt.signal.reason;
1467
+ if (error instanceof FlyApiError &&
1468
+ error.method.toUpperCase() === 'POST' &&
1469
+ error.path === `/apps/${app}/machines` &&
1470
+ error.status >= 400 &&
1471
+ error.status < 500) {
1472
+ try {
1473
+ await this.config.createAttemptJournal?.markMachineCreateFailed(createAttemptId, attempt.generation);
1474
+ }
1475
+ catch (journalError) {
1476
+ logger.warn('Could not record a definitive Fly Machine-create rejection', {
1477
+ app,
1478
+ createAttemptId,
1479
+ error: journalError,
1480
+ });
1481
+ }
1482
+ }
1483
+ throw error;
1484
+ }
1207
1485
  if (!machine?.id) {
1208
1486
  throw new Error(t('codeSandbox.flyio.error.createFailed', { app }, { defaultValue: `Fly Machine creation in app "${app}" returned no Machine id.` }));
1209
1487
  }
1488
+ if (!machineCreateResponseLost) {
1489
+ await this.config.createAttemptJournal?.recordMachineProvenance(createAttemptId, attempt.generation, machine.id);
1490
+ }
1210
1491
  if (manifest && config.templateId) {
1211
- await this.restoreTemplate(app, machine.id, config.templateId, manifest);
1492
+ await this.restoreTemplate(app, machine.id, config.templateId, manifest, attempt);
1212
1493
  // The restore waited for `started` and ran a command in it, so the state
1213
1494
  // read back from the creation response is stale by definition.
1214
1495
  return this.buildSandbox(app, machine.id, 'running');
@@ -1219,35 +1500,7 @@ class FlyioSandboxProvider {
1219
1500
  // `412 failed_precondition: machine not running` until the Machine is up.
1220
1501
  // Wait for `started` here so `create` always hands back a live sandbox, as
1221
1502
  // the JSDoc promises — the template path already does this via restore.
1222
- try {
1223
- await this.waitForStarted(app, machine.id);
1224
- }
1225
- catch (error) {
1226
- // The boot failed after the Machine existed. This call created it and
1227
- // handed it to NOBODY — the caller got a rejection, not a handle — so
1228
- // leaving it behind strands state nothing else ever cleans up: a Machine
1229
- // that reached `started` bills compute until it is reaped, and one that
1230
- // did not keeps the app's single-attach volume claimed (the next
1231
- // create's stale-Machine sweep only runs if this project is retried).
1232
- // The created-app failure path below cannot do this either: it deletes
1233
- // the APP, and `discardCreatedApp` deliberately has no Machine-delete
1234
- // step — an app delete that runs under a live Machine either fails or
1235
- // hides the Machine. Destroy the Machine first (the same order
1236
- // `restoreTemplate`'s failure path and `destroy()` use), then rethrow
1237
- // the boot failure. Best-effort: the boot failure is what the caller
1238
- // must see; a failed cleanup is logged like every other cleanup here.
1239
- try {
1240
- await this.client.request(`/apps/${app}/machines/${machine.id}?force=true`, {
1241
- method: 'DELETE',
1242
- nullOn: [404],
1243
- });
1244
- }
1245
- catch (cleanupError) {
1246
- logger.warn('Failed to destroy the Fly Machine whose boot did not reach started — it will keep ' +
1247
- 'billing (and holding its volume) until it is reaped', { app, machineId: machine.id, error: cleanupError });
1248
- }
1249
- throw error;
1250
- }
1503
+ await this.waitForStarted(app, machine.id, attempt);
1251
1504
  return this.buildSandbox(app, machine.id, 'running');
1252
1505
  }
1253
1506
  /**
@@ -1486,8 +1739,10 @@ class FlyioSandboxProvider {
1486
1739
  'TRUNCATED, so some sandboxes would be missing from the result.',
1487
1740
  }));
1488
1741
  }
1489
- const prefix = `${this.appPrefix()}-`;
1490
- const owned = apps.map((app) => app.name).filter((name) => name?.startsWith(prefix));
1742
+ const namespace = appNameNamespace(this.appPrefix());
1743
+ const owned = apps
1744
+ .map((app) => app.name)
1745
+ .filter((name) => namespace.searchPrefixes.some((prefix) => name?.startsWith(prefix)));
1491
1746
  if (owned.length > FIND_MAX_APPS) {
1492
1747
  throw new Error(t('codeSandbox.flyio.error.findTooManyApps', { count: String(owned.length), max: String(FIND_MAX_APPS) }, {
1493
1748
  defaultValue: `Refusing to enumerate ${owned.length} Fly sandbox apps (cap ${FIND_MAX_APPS}): ` +
@@ -1497,6 +1752,834 @@ class FlyioSandboxProvider {
1497
1752
  }
1498
1753
  return owned;
1499
1754
  }
1755
+ /** Read an existing app's live state; a list 404 after the app read is an error, not absence. */
1756
+ async readAttemptAppState(app, signal) {
1757
+ const [machines, volumes] = await Promise.all([
1758
+ this.requestForDestruction(`/apps/${app}/machines`, { nullOn: [404] }, signal),
1759
+ this.requestForDestruction(`/apps/${app}/volumes`, { nullOn: [404] }, signal),
1760
+ ]);
1761
+ if (Array.isArray(machines) && Array.isArray(volumes)) {
1762
+ return {
1763
+ appGone: false,
1764
+ machines,
1765
+ volumes: volumes.filter((volume) => volume.state !== 'destroyed' && volume.state !== 'destroying'),
1766
+ };
1767
+ }
1768
+ // A Machines/Volumes-list 404 is not proof that a newly-created app is
1769
+ // gone. Confirm the app itself; if it exists, the list read was transient
1770
+ // and responsibility must remain durable.
1771
+ const existing = await this.requestForDestruction(`/apps/${app}`, { nullOn: [404] }, signal);
1772
+ if (existing)
1773
+ throw new Error(`Fly app "${app}" exists but its live-state listing failed`);
1774
+ return { appGone: true, machines: [], volumes: [] };
1775
+ }
1776
+ /** Prove a definitively rejected Machine create has no live consumer before retiring it. */
1777
+ async assertFailedMachineSweepable(intent, signal) {
1778
+ if ((intent.appCreatedByAttemptId === intent.attemptId && intent.appCreateResolved !== true) ||
1779
+ intent.volumes.some((volume) => volume.resolved !== true) ||
1780
+ intent.addresses.some((address) => !address.resolved)) {
1781
+ throw new Error(`Fly create attempt "${intent.attemptId}" still has unresolved ownership`);
1782
+ }
1783
+ const live = await this.readAttemptAppState(intent.appName, signal);
1784
+ if (signal.aborted)
1785
+ throw signal.reason;
1786
+ // Siblings matter only through an actual resource reference below, or at
1787
+ // the later app-deletion boundary. Their mere existence must not strand a
1788
+ // detached, proven-created volume in a shared app.
1789
+ if (live.appGone)
1790
+ return;
1791
+ if (this.appPerProject() && live.machines.length) {
1792
+ throw new Error(`Fly app "${intent.appName}" still has Machines; failed-create sweep is KEEP`);
1793
+ }
1794
+ const volumeIds = new Set(intent.volumes.map((volume) => volume.id).filter((id) => Boolean(id)));
1795
+ for (const machine of live.machines) {
1796
+ if (machine.config?.mounts?.some((mount) => mount.volume && volumeIds.has(mount.volume))) {
1797
+ throw new Error(`Fly Machine "${machine.id}" still references a failed-create volume; sweep is KEEP`);
1798
+ }
1799
+ }
1800
+ for (const volume of live.volumes) {
1801
+ if (volumeIds.has(volume.id) && volume.attached_machine_id) {
1802
+ throw new Error(`Fly volume "${volume.id}" is still attached to "${volume.attached_machine_id}"; sweep is KEEP`);
1803
+ }
1804
+ }
1805
+ }
1806
+ /** Exact durable Machine identity; unlike accepted provenance, sandboxId may be observational. */
1807
+ recordedMachineId(intent) {
1808
+ if (intent.machineId)
1809
+ return intent.machineId;
1810
+ const prefix = `${intent.appName}:`;
1811
+ if (!intent.sandboxId?.startsWith(prefix))
1812
+ return undefined;
1813
+ return intent.sandboxId.slice(prefix.length) || undefined;
1814
+ }
1815
+ /** True when no exact durable Machine ID can settle a possibly-dispatched POST. */
1816
+ hasUnresolvedMachineIdentity(intent) {
1817
+ if (this.recordedMachineId(intent))
1818
+ return false;
1819
+ const legacyUnknown = intent.machineCreateOutcome === undefined &&
1820
+ intent.machineCreateFailed !== true &&
1821
+ !intent.machineId &&
1822
+ !intent.sandboxId &&
1823
+ !intent.machineAdopted;
1824
+ return (legacyUnknown ||
1825
+ intent.machineCreateOutcome === 'pending' ||
1826
+ intent.machineCreateOutcome === 'unknown' ||
1827
+ intent.machineCreateOutcome === 'accepted' ||
1828
+ Boolean(intent.machineId || intent.sandboxId || intent.machineAdopted));
1829
+ }
1830
+ /**
1831
+ * The sole forgetting guard. A listing is never an absence proof: every
1832
+ * durable Machine/volume ID needs this pass's authorized DELETE success/404
1833
+ * or its own exact GET 404. Unresolved identities remain visible as KEEP.
1834
+ */
1835
+ async assertAuthoritativeForget(intent, signal, settled, app) {
1836
+ // A successful non-forced app DELETE (or exact app 404) is authoritative
1837
+ // for every resource contained by that app. External addresses still need
1838
+ // their own settled records below.
1839
+ const containerSettled = app.settled;
1840
+ const machineId = this.recordedMachineId(intent);
1841
+ if (!containerSettled && this.hasUnresolvedMachineIdentity(intent)) {
1842
+ throw new Error(`Fly Machine identity for attempt "${intent.attemptId}" is unresolved; forgetting is KEEP`);
1843
+ }
1844
+ if (!containerSettled && machineId && !settled.machineIds.has(machineId)) {
1845
+ const machine = await this.requestForDestruction(`/apps/${intent.appName}/machines/${machineId}`, { nullOn: [404] }, signal);
1846
+ if (machine) {
1847
+ throw new Error(`Fly Machine "${machineId}" for attempt "${intent.attemptId}" still exists; forgetting is KEEP`);
1848
+ }
1849
+ }
1850
+ for (const volume of containerSettled ? [] : intent.volumes) {
1851
+ if (volume.resolved !== true) {
1852
+ throw new Error(`Fly volume "${volume.id ?? volume.name}" for attempt "${intent.attemptId}" is unresolved; forgetting is KEEP`);
1853
+ }
1854
+ if (!volume.id || settled.volumeIds.has(volume.id))
1855
+ continue;
1856
+ const live = await this.requestForDestruction(`/apps/${intent.appName}/volumes/${volume.id}`, { nullOn: [404] }, signal);
1857
+ if (live) {
1858
+ throw new Error(`Fly volume "${volume.id}" for attempt "${intent.attemptId}" still exists; forgetting is KEEP`);
1859
+ }
1860
+ }
1861
+ if (intent.addresses.some((address) => !address.resolved || address.owned)) {
1862
+ throw new Error(`Fly addresses for attempt "${intent.attemptId}" are not settled; KEEP`);
1863
+ }
1864
+ if (app.required && !app.settled) {
1865
+ throw new Error(`Fly app "${intent.appName}" has no authoritative absence result; KEEP`);
1866
+ }
1867
+ }
1868
+ /**
1869
+ * The ONE automatic deletion/settlement function. Its four invariants are:
1870
+ * (a) it runs only under the owning attempt's lifecycle lock, and re-reads the
1871
+ * durable record and Fly live state inside that lock before EVERY delete;
1872
+ * (b) automatic deletion additionally requires this attempt's accepted-POST
1873
+ * provenance and exact resource ID; adoption is never deletion authority;
1874
+ * (c) unknown outcomes and transient absence retain responsibility. The one
1875
+ * forgetting boundary below requires an authoritative exact-ID result for
1876
+ * every resource: an exact GET 404, or an authorized DELETE success/404. A
1877
+ * listing, empty listing, NULL ID or "no recorded resources" proves no
1878
+ * absence; an unresolved identity keeps the row and reports KEEP. Durable
1879
+ * addresses are released before app deletion/forgetting; and (d) cancellation
1880
+ * plus the attempt's current generation fences every Fly mutation. A stale,
1881
+ * serving or unreadable attempt therefore cannot delete anything or be forgotten.
1882
+ */
1883
+ async settleCreateAttempt(snapshot, signal, generation, options = {}) {
1884
+ const automatic = options.automatic ?? false;
1885
+ const retire = options.retire ?? false;
1886
+ const failedMachineSweep = options.failedMachineSweep ?? false;
1887
+ const deleteApp = options.deleteApp ?? true;
1888
+ const forget = options.forget ?? true;
1889
+ const appRequired = deleteApp && this.appPerProject();
1890
+ // Automatic reconciliation never deletes an app: a container DELETE can
1891
+ // cascade into any sibling's retained or not-yet-resolved user resources.
1892
+ // An existing app remains a durable KEEP for deliberate retirement.
1893
+ const mayDeleteApp = appRequired && !automatic;
1894
+ const journal = this.config.createAttemptJournal;
1895
+ if (!journal)
1896
+ throw new Error('A durable create-attempt journal is required for settlement');
1897
+ const fresh = async () => {
1898
+ if (signal.aborted)
1899
+ throw signal.reason;
1900
+ await journal.assertGeneration(snapshot.attemptId, generation);
1901
+ const current = await journal.get(snapshot.attemptId);
1902
+ if (signal.aborted)
1903
+ throw signal.reason;
1904
+ if (!current || (current.generation ?? 1) !== generation) {
1905
+ throw new Error(`Fly create attempt "${snapshot.attemptId}" was superseded`);
1906
+ }
1907
+ if (current.state !== 'failed' && current.state !== 'abandoned') {
1908
+ throw new Error(`Fly create attempt "${snapshot.attemptId}" is no longer settleable`);
1909
+ }
1910
+ if ((options.expectedAppName && current.appName !== options.expectedAppName) ||
1911
+ (options.expectedMachineId && this.recordedMachineId(current) !== options.expectedMachineId)) {
1912
+ throw new Error(`Fly create attempt "${snapshot.attemptId}" changed outside the caller-validated scope`);
1913
+ }
1914
+ return current;
1915
+ };
1916
+ const ownMachine = (machines, intent) => {
1917
+ // Once an exact ID is durable, a same-name Machine is potentially a
1918
+ // replacement and must never inherit that identity. machineId may be an
1919
+ // observation; only machineCreatedByAttemptId grants automatic authority.
1920
+ const recordedId = this.recordedMachineId(intent);
1921
+ if (recordedId)
1922
+ return machines.find((machine) => machine.id === recordedId);
1923
+ if (options.recordedResourcesOnly)
1924
+ return undefined;
1925
+ const direct = machines.find((machine) => machine.name === intent.attemptId ||
1926
+ machine.config?.metadata?.[`${this.metadataPrefix}.createAttempt`] === intent.attemptId);
1927
+ if (direct || !intent.attemptId.startsWith('legacy:') || !intent.sandboxId)
1928
+ return direct;
1929
+ const prefix = `${intent.appName}:`;
1930
+ if (!intent.sandboxId.startsWith(prefix))
1931
+ return undefined;
1932
+ const sandboxMachineId = intent.sandboxId.slice(prefix.length);
1933
+ return machines.find((machine) => machine.id === sandboxMachineId);
1934
+ };
1935
+ const adoptLegacyResources = async (machine) => {
1936
+ if (!intent.attemptId.startsWith('legacy:'))
1937
+ return;
1938
+ const metadata = machine.config?.metadata ?? {};
1939
+ const logicalName = metadata[`${this.metadataPrefix}.volumeName`];
1940
+ for (const mount of machine.config?.mounts ?? []) {
1941
+ if (!mount.volume || intent.volumes.some((volume) => volume.id === mount.volume))
1942
+ continue;
1943
+ const listed = live.volumes.find((volume) => volume.id === mount.volume);
1944
+ await journal.recordVolume(intent.attemptId, {
1945
+ id: mount.volume,
1946
+ name: listed?.name ?? mount.volume,
1947
+ ...(logicalName ? { logicalName } : {}),
1948
+ workspace: false,
1949
+ resolved: true,
1950
+ adoptedByAttemptId: intent.attemptId,
1951
+ });
1952
+ }
1953
+ const routes = decodePrivateRoutes(metadata[`${this.metadataPrefix}.${PRIVATE_ROUTES_METADATA_SUFFIX}`]);
1954
+ const network = this.networkFor(intent.appName);
1955
+ for (const [targetApp, ip] of Object.entries(routes)) {
1956
+ // A durable exact binding for the intended network is authoritative.
1957
+ // Otherwise validate the IP independently; copied metadata is only a hint.
1958
+ if (intent.addresses.some((address) => address.targetApp === targetApp &&
1959
+ address.network === network &&
1960
+ address.ip === ip &&
1961
+ address.resolved)) {
1962
+ continue;
1963
+ }
1964
+ const validated = await this.privateRouteMatchesNetwork(targetApp, ip, network, signal);
1965
+ await journal.recordAddress(intent.attemptId, {
1966
+ targetApp,
1967
+ network: network ?? intent.appName,
1968
+ ip,
1969
+ owned: validated,
1970
+ resolved: validated,
1971
+ });
1972
+ }
1973
+ };
1974
+ const assertObservedMachineUnreferenced = async (intent, machineId) => {
1975
+ if (intent.machineCreatedByAttemptId === intent.attemptId)
1976
+ return;
1977
+ const siblings = await journal.forApp(intent.appName);
1978
+ if (signal.aborted)
1979
+ throw signal.reason;
1980
+ const conflict = siblings.find((candidate) => candidate.attemptId !== intent.attemptId &&
1981
+ candidate.state !== 'abandoned' &&
1982
+ this.recordedMachineId(candidate) === machineId);
1983
+ if (conflict) {
1984
+ throw new Error(`Fly Machine "${machineId}" is still referenced by ${conflict.state} attempt "${conflict.attemptId}"; observed retirement is KEEP`);
1985
+ }
1986
+ };
1987
+ const activeStates = [
1988
+ 'creating',
1989
+ 'starting',
1990
+ 'started',
1991
+ 'restarting',
1992
+ 'updating',
1993
+ 'replacing',
1994
+ 'migrated',
1995
+ ];
1996
+ const hasActiveSibling = async (intent) => {
1997
+ if (signal.aborted)
1998
+ throw signal.reason;
1999
+ const siblings = await journal.forApp(intent.appName);
2000
+ if (signal.aborted)
2001
+ throw signal.reason;
2002
+ return siblings.some((candidate) => {
2003
+ if (candidate.attemptId === intent.attemptId)
2004
+ return false;
2005
+ if (candidate.state === 'serving')
2006
+ return true;
2007
+ // Every expired sibling in this set is independently reclaimable and
2008
+ // will be retired under its own lock in this pass. The current app lock
2009
+ // fences it now; treating the stale row as live deadlocks both rows.
2010
+ return (candidate.state === 'creating' && !options.expiredCreatingIds?.has(candidate.attemptId));
2011
+ });
2012
+ };
2013
+ const readRecordedMachine = async (current) => {
2014
+ const id = this.recordedMachineId(current);
2015
+ if (!id)
2016
+ return undefined;
2017
+ return this.requestForDestruction(`/apps/${current.appName}/machines/${id}`, { nullOn: [404] }, signal);
2018
+ };
2019
+ const releasedMachineIds = new Set();
2020
+ let intent = await fresh();
2021
+ let live = await this.readAttemptAppState(intent.appName, signal);
2022
+ const observed = ownMachine(options.observedMachines ?? [], intent);
2023
+ if (observed) {
2024
+ // destroyApp read this metadata before waiting for the attempt lock. Save
2025
+ // that observation now; a later list omission cannot erase ownership.
2026
+ intent = await fresh();
2027
+ await adoptLegacyResources(observed);
2028
+ intent = await fresh();
2029
+ }
2030
+ // A list omission cannot erase a durable ID. Read that exact resource;
2031
+ // 403/5xx keeps the row, 404 is authoritative, and a live result follows
2032
+ // the same state/provenance checks as a listed Machine.
2033
+ const machine = ownMachine(live.machines, intent) ??
2034
+ (!forget || mayDeleteApp ? undefined : ((await readRecordedMachine(intent)) ?? undefined));
2035
+ if (machine && !this.recordedMachineId(intent)) {
2036
+ // A name/metadata match recovers an exact identity as observation only.
2037
+ // It deliberately does not gain automatic deletion provenance.
2038
+ await journal.recordMachineObservation(intent.attemptId, generation, machine.id);
2039
+ intent = await fresh();
2040
+ }
2041
+ if (automatic &&
2042
+ machine &&
2043
+ (intent.machineCreatedByAttemptId !== intent.attemptId || intent.machineId !== machine.id)) {
2044
+ throw new Error(`Fly Machine "${machine.id}" for attempt "${intent.attemptId}" has no matching accepted-POST provenance; KEEP`);
2045
+ }
2046
+ if (failedMachineSweep && machine) {
2047
+ throw new Error(`Fly Machine "${machine.id}" appeared during failed-create sweep; KEEP`);
2048
+ }
2049
+ if (!retire &&
2050
+ machine &&
2051
+ (activeStates.includes(machine.state ?? 'created') ||
2052
+ intent.volumes.some((volume) => volume.workspace))) {
2053
+ return false;
2054
+ }
2055
+ if (machine) {
2056
+ // Re-read immediately before the destructive decision, not from the lock-wait snapshot.
2057
+ intent = await fresh();
2058
+ live = await this.readAttemptAppState(intent.appName, signal);
2059
+ const current = ownMachine(live.machines, intent) ??
2060
+ (!forget || mayDeleteApp ? undefined : ((await readRecordedMachine(intent)) ?? undefined));
2061
+ if (failedMachineSweep && current) {
2062
+ throw new Error(`Fly Machine "${current.id}" appeared during failed-create sweep; KEEP`);
2063
+ }
2064
+ if (!retire &&
2065
+ current &&
2066
+ (activeStates.includes(current.state ?? 'created') ||
2067
+ intent.volumes.some((volume) => volume.workspace))) {
2068
+ return false;
2069
+ }
2070
+ if (automatic &&
2071
+ current &&
2072
+ (intent.machineCreatedByAttemptId !== intent.attemptId || intent.machineId !== current.id)) {
2073
+ throw new Error(`Fly Machine "${current.id}" for attempt "${intent.attemptId}" lost accepted-POST provenance before deletion; KEEP`);
2074
+ }
2075
+ if (current) {
2076
+ // Migration 1793275400000 recorded only the synthetic attempt and
2077
+ // sandbox ID. The owned Machine's mounts/routes rebuild the exact-ID
2078
+ // responsibilities before its metadata disappears on deletion.
2079
+ await adoptLegacyResources(current);
2080
+ intent = await fresh();
2081
+ // Observation is not ownership. Recheck every other durable reference
2082
+ // on this same app lock immediately before the irreversible request.
2083
+ await assertObservedMachineUnreferenced(intent, current.id);
2084
+ await this.deleteRetired(intent, generation, `/apps/${intent.appName}/machines/${current.id}?force=true`, signal);
2085
+ releasedMachineIds.add(current.id);
2086
+ }
2087
+ }
2088
+ const releasedVolumeIds = new Set();
2089
+ for (const recorded of intent.volumes) {
2090
+ intent = await fresh();
2091
+ let volumeIntent = intent.volumes.find((candidate) => candidate.name === recorded.name || candidate.id === recorded.id);
2092
+ if (!volumeIntent || (volumeIntent.resolved === true && !volumeIntent.id))
2093
+ continue;
2094
+ live = await this.readAttemptAppState(intent.appName, signal);
2095
+ if (live.appGone)
2096
+ continue;
2097
+ if (options.recordedResourcesOnly && !volumeIntent.id) {
2098
+ throw new Error(`Fly volume "${volumeIntent.name}" for attempt "${intent.attemptId}" has no exact durable ID; refusing discovery during explicit retirement`);
2099
+ }
2100
+ if (!volumeIntent.id) {
2101
+ const matches = live.volumes.filter((candidate) => candidate.name === volumeIntent?.name);
2102
+ if (matches.length > 1) {
2103
+ throw new Error(`Fly volume create "${volumeIntent.name}" has ambiguous matches; an explicit ID is required`);
2104
+ }
2105
+ const discovered = matches[0];
2106
+ if (!discovered) {
2107
+ throw new Error(`Fly volume create "${volumeIntent.name}" remains unresolved`);
2108
+ }
2109
+ // A unique name match is an observation only. Persist its exact ID for
2110
+ // deliberate retirement, never accepted-POST/automatic-delete authority.
2111
+ await journal.recordVolume(intent.attemptId, {
2112
+ ...volumeIntent,
2113
+ id: discovered.id,
2114
+ resolved: true,
2115
+ createdByAttemptId: undefined,
2116
+ adoptedByAttemptId: intent.attemptId,
2117
+ });
2118
+ intent = await fresh();
2119
+ volumeIntent = intent.volumes.find((candidate) => candidate.id === discovered.id);
2120
+ }
2121
+ if (automatic &&
2122
+ volumeIntent &&
2123
+ (volumeIntent.createdByAttemptId !== intent.attemptId || volumeIntent.resolved !== true)) {
2124
+ throw new Error(`Fly volume "${volumeIntent.id ?? volumeIntent.name}" for attempt "${intent.attemptId}" has no matching accepted-POST provenance; KEEP`);
2125
+ }
2126
+ if (!volumeIntent?.id || (!retire && volumeIntent.workspace))
2127
+ continue;
2128
+ // ID, attachment, workspace flag and ownership are all re-read under the lock.
2129
+ intent = await fresh();
2130
+ volumeIntent = intent.volumes.find((candidate) => candidate.id === volumeIntent?.id);
2131
+ live = await this.readAttemptAppState(intent.appName, signal);
2132
+ const volume = live.volumes.find((candidate) => candidate.id === volumeIntent?.id);
2133
+ if (failedMachineSweep && volumeIntent?.id) {
2134
+ const referencing = live.machines.find((machine) => machine.config?.mounts?.some((mount) => mount.volume === volumeIntent?.id));
2135
+ if (referencing || volume?.attached_machine_id) {
2136
+ throw new Error(`Fly volume "${volumeIntent.id}" gained a Machine reference during sweep; KEEP`);
2137
+ }
2138
+ }
2139
+ if (automatic &&
2140
+ volumeIntent &&
2141
+ (volumeIntent.createdByAttemptId !== intent.attemptId || volumeIntent.resolved !== true)) {
2142
+ throw new Error(`Fly volume "${volumeIntent.id ?? volumeIntent.name}" for attempt "${intent.attemptId}" lost accepted-POST provenance before deletion; KEEP`);
2143
+ }
2144
+ if (!volumeIntent?.id || (!retire && volumeIntent.workspace) || volume?.attached_machine_id) {
2145
+ continue;
2146
+ }
2147
+ // A list omission is not deletion. The durable exact ID is sufficient
2148
+ // capability for an idempotent DELETE; success/404, never the list,
2149
+ // authorizes forgetting it. Fly still rejects a really attached volume.
2150
+ const volumeId = volumeIntent.id;
2151
+ await this.deleteRetired(intent, generation, `/apps/${intent.appName}/volumes/${volumeId}`, signal);
2152
+ releasedVolumeIds.add(volumeId);
2153
+ }
2154
+ intent = await fresh();
2155
+ live = await this.readAttemptAppState(intent.appName, signal);
2156
+ if (intent.addresses.length &&
2157
+ (live.machines.some((candidate) => candidate.id !== machine?.id) ||
2158
+ (await hasActiveSibling(intent)))) {
2159
+ return false;
2160
+ }
2161
+ for (const recorded of intent.addresses) {
2162
+ intent = await fresh();
2163
+ let address = intent.addresses.find((candidate) => candidate.targetApp === recorded.targetApp && candidate.network === recorded.network);
2164
+ if (!address || (address.resolved && !address.owned))
2165
+ continue;
2166
+ if (options.recordedResourcesOnly && (!address.owned || !address.ip)) {
2167
+ throw new Error(`Fly address for attempt "${intent.attemptId}" on "${address.targetApp}" has no exact owned IP; refusing discovery during explicit retirement`);
2168
+ }
2169
+ if (!address.resolved) {
2170
+ // Only an unresolved create needs discovery. A known owned IP is its
2171
+ // own DELETE capability, even when the target app no longer exists.
2172
+ const assignments = await this.listPrivateAssignments(address.targetApp, signal);
2173
+ if (!assignments) {
2174
+ throw new Error(`Could not resolve Fly addresses on "${address.targetApp}"`);
2175
+ }
2176
+ const assignment = assignments.find((candidate) => candidate.network === address?.network);
2177
+ if (!assignment?.ip) {
2178
+ throw new Error(`Fly address allocation for attempt "${intent.attemptId}" is unresolved`);
2179
+ }
2180
+ await journal.recordAddress(intent.attemptId, {
2181
+ ...address,
2182
+ ip: assignment.ip,
2183
+ owned: true,
2184
+ resolved: true,
2185
+ });
2186
+ intent = await fresh();
2187
+ address = intent.addresses.find((candidate) => candidate.targetApp === recorded.targetApp && candidate.network === recorded.network);
2188
+ }
2189
+ if (!address?.owned)
2190
+ continue;
2191
+ if (!address.ip)
2192
+ throw new Error(`Owned Fly address for "${intent.attemptId}" has no IP`);
2193
+ // The durable exact IP, not list presence, is the release capability.
2194
+ // A retry after a lost successful DELETE gets 404 and completes below.
2195
+ intent = await fresh();
2196
+ address = intent.addresses.find((candidate) => candidate.targetApp === recorded.targetApp && candidate.network === recorded.network);
2197
+ if (!address?.owned)
2198
+ continue;
2199
+ if (!address.ip)
2200
+ throw new Error(`Owned Fly address for "${intent.attemptId}" has no IP`);
2201
+ await this.deleteRetired(intent, generation, `/apps/${address.targetApp}/ip_assignments/${encodeURIComponent(address.ip)}`, signal);
2202
+ await journal.recordAddress(intent.attemptId, {
2203
+ ...address,
2204
+ owned: false,
2205
+ resolved: true,
2206
+ });
2207
+ }
2208
+ intent = await fresh();
2209
+ live = await this.readAttemptAppState(intent.appName, signal);
2210
+ const ownedVolumesRemain = intent.volumes.some((volume) => {
2211
+ if (volume.resolved === true && !volume.id)
2212
+ return false;
2213
+ if (live.appGone)
2214
+ return false;
2215
+ if (volume.id)
2216
+ return !releasedVolumeIds.has(volume.id);
2217
+ return live.volumes.some((candidate) => candidate.name === volume.name);
2218
+ });
2219
+ // Exact app 404 proves absence of every contained resource, including an
2220
+ // unresolved volume. External addresses still settle independently below.
2221
+ const unsettledVolume = !live.appGone && intent.volumes.some((volume) => volume.resolved !== true);
2222
+ if (failedMachineSweep && this.appPerProject() && live.machines.length) {
2223
+ throw new Error(`Fly app "${intent.appName}" gained a Machine during sweep; KEEP`);
2224
+ }
2225
+ if (failedMachineSweep && deleteApp && this.appPerProject() && live.volumes.length) {
2226
+ throw new Error(`Fly app "${intent.appName}" is not empty; failed-create sweep is KEEP`);
2227
+ }
2228
+ const listedMachine = ownMachine(live.machines, intent);
2229
+ if (unsettledVolume ||
2230
+ ownedVolumesRemain ||
2231
+ (listedMachine && !releasedMachineIds.has(listedMachine.id))) {
2232
+ return false;
2233
+ }
2234
+ if (intent.addresses.some((address) => !address.resolved || address.owned))
2235
+ return false;
2236
+ if (mayDeleteApp && !live.appGone && (await hasActiveSibling(intent))) {
2237
+ return false;
2238
+ }
2239
+ let appSettled = live.appGone;
2240
+ if (mayDeleteApp && !live.appGone) {
2241
+ // Deliberate retirement alone may delete a container. Addresses above are
2242
+ // settled first; provenance and every Machine are re-read under the lock.
2243
+ intent = await fresh();
2244
+ live = await this.readAttemptAppState(intent.appName, signal);
2245
+ if (failedMachineSweep && (live.machines.length || live.volumes.length)) {
2246
+ throw new Error(`Fly app "${intent.appName}" is not empty before app deletion; KEEP`);
2247
+ }
2248
+ if (automatic &&
2249
+ (intent.appCreatedByAttemptId !== intent.attemptId || intent.appCreateResolved !== true)) {
2250
+ throw new Error(`Fly app "${intent.appName}" for attempt "${intent.attemptId}" has no matching accepted-POST provenance; KEEP`);
2251
+ }
2252
+ if (live.machines.length || ((automatic || !retire) && live.volumes.length))
2253
+ return false;
2254
+ await this.deleteRetired(intent, generation, `/apps/${intent.appName}`, signal);
2255
+ appSettled = true;
2256
+ }
2257
+ if (automatic && appRequired && !appSettled && !this.hasUnresolvedMachineIdentity(intent)) {
2258
+ return false;
2259
+ }
2260
+ if (!forget)
2261
+ return false;
2262
+ intent = await fresh();
2263
+ await this.assertAuthoritativeForget(intent, signal, { machineIds: releasedMachineIds, volumeIds: releasedVolumeIds }, { required: appRequired, settled: appSettled });
2264
+ await journal.forget(intent.attemptId, generation);
2265
+ return true;
2266
+ }
2267
+ /** Lock and settle each eligible attempt independently; there is no lock-free mode. */
2268
+ async reconcileCreateAttempts(options = {}) {
2269
+ const journal = this.config.createAttemptJournal;
2270
+ if (!journal)
2271
+ return { deleted: 0, failed: 0 };
2272
+ const now = options.now ?? Date.now();
2273
+ const timeoutMs = options.createTimeoutMs ?? 15 * 60_000;
2274
+ const cutoff = new Date(now - timeoutMs);
2275
+ const intents = (await journal.reclaimable(cutoff)).filter((intent) => !options.attemptId || intent.attemptId === options.attemptId);
2276
+ const expiredCreatingIds = new Set(intents.filter((intent) => intent.state === 'creating').map((intent) => intent.attemptId));
2277
+ const summary = { deleted: 0, failed: 0 };
2278
+ for (const intent of intents) {
2279
+ try {
2280
+ if (!journal.withLock)
2281
+ throw new Error('Attempt settlement requires its lifecycle lock');
2282
+ const forgotten = await journal.withLock(intent, async (signal, generation) => {
2283
+ // Selection happened before the lock wait. Re-read state and expiry;
2284
+ // only failed/expired attempts may be retired by this automatic pass.
2285
+ const current = await journal.get(intent.attemptId);
2286
+ if (!current)
2287
+ return false;
2288
+ if (current.state === 'serving')
2289
+ return false;
2290
+ if (current.state === 'creating') {
2291
+ const expired = (await journal.reclaimable(cutoff)).some((candidate) => candidate.attemptId === intent.attemptId);
2292
+ if (!expired)
2293
+ return false;
2294
+ }
2295
+ if (signal.aborted)
2296
+ throw signal.reason;
2297
+ const failedMachineSweep = current.machineCreateFailed === true;
2298
+ if (failedMachineSweep)
2299
+ await this.assertFailedMachineSweepable(current, signal);
2300
+ if (current.state === 'failed' || current.state === 'creating') {
2301
+ await journal.retire(current.attemptId, generation);
2302
+ }
2303
+ if (signal.aborted)
2304
+ throw signal.reason;
2305
+ const settled = await this.settleCreateAttempt(current, signal, generation, {
2306
+ automatic: true,
2307
+ retire: failedMachineSweep,
2308
+ failedMachineSweep,
2309
+ expiredCreatingIds,
2310
+ });
2311
+ if (failedMachineSweep && !settled) {
2312
+ throw new Error(`Fly failed-create sweep for "${current.attemptId}" remains incomplete`);
2313
+ }
2314
+ return settled;
2315
+ });
2316
+ if (forgotten)
2317
+ summary.deleted++;
2318
+ }
2319
+ catch (error) {
2320
+ // A state transition back to creating/serving is a safe skip, not a cleanup failure.
2321
+ if (!/no longer settleable/.test(error instanceof Error ? error.message : '')) {
2322
+ summary.failed++;
2323
+ logger.warn('Could not reconcile a Fly create attempt; retaining its durable intent', {
2324
+ attemptId: intent.attemptId,
2325
+ app: intent.appName,
2326
+ error,
2327
+ });
2328
+ }
2329
+ }
2330
+ }
2331
+ return summary;
2332
+ }
2333
+ /** Re-read and abandon only while the attempt's current lifecycle lock is live. */
2334
+ async retireCreateAttempt(snapshot, outerSignal, options) {
2335
+ const journal = this.config.createAttemptJournal;
2336
+ if (!journal?.withLock)
2337
+ throw new Error('Attempt settlement requires its lifecycle lock');
2338
+ return journal.withLock(snapshot, async (lockSignal, generation) => {
2339
+ const signal = AbortSignal.any([outerSignal, lockSignal]);
2340
+ if (signal.aborted)
2341
+ throw signal.reason;
2342
+ const current = await journal.get(snapshot.attemptId);
2343
+ if (signal.aborted)
2344
+ throw signal.reason;
2345
+ if (!current)
2346
+ return false;
2347
+ if ((options.expectedAppName && current.appName !== options.expectedAppName) ||
2348
+ (options.expectedMachineId && this.recordedMachineId(current) !== options.expectedMachineId)) {
2349
+ throw new Error(`Fly create attempt "${snapshot.attemptId}" changed outside the caller-validated scope`);
2350
+ }
2351
+ if (current.state !== 'abandoned') {
2352
+ if (signal.aborted)
2353
+ throw signal.reason;
2354
+ await journal.retire(current.attemptId, generation);
2355
+ if (signal.aborted)
2356
+ throw signal.reason;
2357
+ }
2358
+ return this.settleCreateAttempt(current, signal, generation, {
2359
+ retire: true,
2360
+ ...options,
2361
+ });
2362
+ });
2363
+ }
2364
+ /** Journal a pre-journal Machine before its first destructive request. */
2365
+ async journalLegacyMachine(app, machine, fallbackProjectId) {
2366
+ const journal = this.config.createAttemptJournal;
2367
+ if (!journal)
2368
+ throw new Error('Legacy Machine adoption requires its durable journal');
2369
+ const metadata = machine.config?.metadata ?? {};
2370
+ let projectId = metadata[`${this.metadataPrefix}.projectId`] ?? fallbackProjectId;
2371
+ if (!projectId && this.appPerProject()) {
2372
+ const prefix = `${this.appPrefix()}-`;
2373
+ const candidate = app.startsWith(prefix) ? app.slice(prefix.length) : '';
2374
+ if (candidate && appNameForProject(this.appPrefix(), candidate) === app) {
2375
+ projectId = candidate;
2376
+ }
2377
+ }
2378
+ if (!projectId) {
2379
+ throw new Error(`Refusing to destroy legacy Machine "${machine.id}" without its project identity`);
2380
+ }
2381
+ const sandboxId = `${app}:${machine.id}`;
2382
+ const attemptId = `legacy:${sandboxId}`;
2383
+ const existing = await journal.get(attemptId);
2384
+ if (existing) {
2385
+ await journal.markMachineAdopted(attemptId);
2386
+ if (existing.sandboxId) {
2387
+ const adopted = await journal.get(attemptId);
2388
+ if (!adopted)
2389
+ throw new Error(`Could not persist legacy ownership for "${sandboxId}"`);
2390
+ return adopted;
2391
+ }
2392
+ await journal.markState(attemptId, existing.state === 'creating' ? 'serving' : existing.state, sandboxId);
2393
+ const repaired = await journal.get(attemptId);
2394
+ if (!repaired?.sandboxId) {
2395
+ throw new Error(`Could not repair legacy ownership for "${sandboxId}"`);
2396
+ }
2397
+ return repaired;
2398
+ }
2399
+ await journal.begin({
2400
+ attemptId,
2401
+ projectId,
2402
+ ownerProjectId: metadata['molecule.production'] ?? projectId,
2403
+ appName: app,
2404
+ state: 'serving',
2405
+ sandboxId,
2406
+ machineAdopted: true,
2407
+ });
2408
+ const adopted = await journal.get(attemptId);
2409
+ if (!adopted)
2410
+ throw new Error(`Could not persist legacy ownership for "${sandboxId}"`);
2411
+ return adopted;
2412
+ }
2413
+ /** Journal an exclusively owned empty app rather than treating absence as DELETE authority. */
2414
+ async journalLegacyApp(app, projectId) {
2415
+ const journal = this.config.createAttemptJournal;
2416
+ if (!journal)
2417
+ throw new Error('Legacy app adoption requires its durable journal');
2418
+ const attemptId = `legacy:${app}:app`;
2419
+ const existing = await journal.get(attemptId);
2420
+ if (existing)
2421
+ return existing;
2422
+ await journal.begin({ attemptId, projectId, appName: app });
2423
+ await journal.markState(attemptId, 'serving');
2424
+ const adopted = await journal.get(attemptId);
2425
+ if (!adopted)
2426
+ throw new Error(`Could not persist legacy ownership for app "${app}"`);
2427
+ return adopted;
2428
+ }
2429
+ /**
2430
+ * Explicitly destroy an exclusively owned app and its workspace storage.
2431
+ * Recorded external addresses are released before any intent may be forgotten.
2432
+ */
2433
+ async destroyApp(projectId, expectedAppName, signal) {
2434
+ if (!signal)
2435
+ throw new Error('Fly app destruction requires its lifecycle lock signal');
2436
+ if (!this.appPerProject()) {
2437
+ throw new Error('Refusing to delete a shared Fly app as though a caller exclusively owned it');
2438
+ }
2439
+ const app = appNameForProject(this.appPrefix(), projectId);
2440
+ if (expectedAppName !== undefined && expectedAppName !== app) {
2441
+ throw new Error(`Refusing to delete Fly app "${expectedAppName}": its durable identity resolves to "${app}"`);
2442
+ }
2443
+ const listed = await this.requestForDestruction(`/apps/${app}/machines`, { nullOn: [404] }, signal);
2444
+ const machines = Array.isArray(listed) ? listed : [];
2445
+ const routes = {};
2446
+ const attemptIds = new Set();
2447
+ const metadataHints = [];
2448
+ for (const machine of machines) {
2449
+ Object.assign(routes, decodePrivateRoutes(machine.config?.metadata?.[`${this.metadataPrefix}.${PRIVATE_ROUTES_METADATA_SUFFIX}`]));
2450
+ const attemptId = machine.config?.metadata?.[`${this.metadataPrefix}.createAttempt`];
2451
+ if (attemptId) {
2452
+ attemptIds.add(attemptId);
2453
+ metadataHints.push({ attemptId, machineId: machine.id });
2454
+ }
2455
+ }
2456
+ const journal = this.config.createAttemptJournal;
2457
+ if (journal) {
2458
+ const durable = new Map();
2459
+ const expectedMachineIds = new Map();
2460
+ const journaled = await journal.forApp(app);
2461
+ if (signal.aborted)
2462
+ throw signal.reason;
2463
+ for (const intent of journaled)
2464
+ durable.set(intent.attemptId, intent);
2465
+ for (const hint of metadataHints) {
2466
+ const intent = await journal.get(hint.attemptId);
2467
+ if (signal.aborted)
2468
+ throw signal.reason;
2469
+ // Metadata is only a hint. Both the requested app and this listed
2470
+ // Machine's exact durable identity must match before selection.
2471
+ if (intent?.appName === app && this.recordedMachineId(intent) === hint.machineId) {
2472
+ durable.set(intent.attemptId, intent);
2473
+ expectedMachineIds.set(intent.attemptId, hint.machineId);
2474
+ }
2475
+ else if (intent) {
2476
+ throw new Error(`Refusing copied createAttempt metadata for Machine "${hint.machineId}" in app "${app}"`);
2477
+ }
2478
+ }
2479
+ for (const machine of machines) {
2480
+ const createAttemptId = machine.config?.metadata?.[`${this.metadataPrefix}.createAttempt`];
2481
+ const known = [...durable.values()].some((intent) => intent.appName === app &&
2482
+ ((intent.attemptId === createAttemptId &&
2483
+ this.recordedMachineId(intent) === machine.id) ||
2484
+ intent.sandboxId === `${app}:${machine.id}`));
2485
+ if (known)
2486
+ continue;
2487
+ const adopted = await this.journalLegacyMachine(app, machine, projectId);
2488
+ durable.set(adopted.attemptId, adopted);
2489
+ }
2490
+ if (!durable.size) {
2491
+ const adopted = await this.journalLegacyApp(app, projectId);
2492
+ durable.set(adopted.attemptId, adopted);
2493
+ }
2494
+ if (durable.size && !journal.withLock) {
2495
+ throw new Error(`Refusing to retire Fly app "${app}" without its lifecycle lock`);
2496
+ }
2497
+ if (durable.size && journal.withLock) {
2498
+ // Pass one settles every durable address and retires Machines without
2499
+ // deleting the app. No app deletion can bypass another attempt's failed
2500
+ // address release. Pass two deletes the app and forgets settled intents.
2501
+ for (const deleteApp of [false, true]) {
2502
+ for (const snapshot of durable.values()) {
2503
+ const current = await journal.get(snapshot.attemptId);
2504
+ if (signal.aborted)
2505
+ throw signal.reason;
2506
+ if (!current)
2507
+ continue;
2508
+ await this.retireCreateAttempt(current, signal, {
2509
+ deleteApp,
2510
+ forget: deleteApp,
2511
+ observedMachines: machines,
2512
+ expectedAppName: app,
2513
+ expectedMachineId: expectedMachineIds.get(current.attemptId),
2514
+ });
2515
+ }
2516
+ }
2517
+ const retained = (await Promise.all([...durable.keys()].map(async (attemptId) => journal.get(attemptId)))).some(Boolean);
2518
+ if (signal.aborted)
2519
+ throw signal.reason;
2520
+ if (retained)
2521
+ throw new Error(`Fly app "${app}" retirement has not fully settled`);
2522
+ return;
2523
+ }
2524
+ }
2525
+ // Legacy unjournaled apps retain explicit destruction. Never force an app
2526
+ // through a Machine that has no committed retired row: explicitly delete
2527
+ // each observed Machine, release its routes, then re-list before app DELETE.
2528
+ for (const machine of machines) {
2529
+ await this.requestForDestruction(`/apps/${app}/machines/${machine.id}?force=true`, { method: 'DELETE', nullOn: [404] }, signal);
2530
+ }
2531
+ for (const [targetApp, ip] of Object.entries(routes)) {
2532
+ if (!(await this.privateRouteMatchesNetwork(targetApp, ip, this.networkFor(app), signal))) {
2533
+ logger.warn('Kept a Fly private (Flycast) address with unvalidated network ownership', {
2534
+ targetApp,
2535
+ ip,
2536
+ app,
2537
+ });
2538
+ continue;
2539
+ }
2540
+ await this.requestForDestruction(`/apps/${targetApp}/ip_assignments/${encodeURIComponent(ip)}`, { method: 'DELETE', nullOn: [404] }, signal);
2541
+ }
2542
+ const remaining = await this.readAttemptAppState(app, signal);
2543
+ if (remaining.machines.length) {
2544
+ throw new Error(`Refusing to delete Fly app "${app}" while a Machine remains`);
2545
+ }
2546
+ if (!remaining.appGone) {
2547
+ await this.requestForDestruction(`/apps/${app}`, { method: 'DELETE', nullOn: [404] }, signal);
2548
+ }
2549
+ for (const attemptId of attemptIds) {
2550
+ if (signal.aborted)
2551
+ throw signal.reason;
2552
+ await journal?.forget(attemptId);
2553
+ }
2554
+ }
2555
+ /**
2556
+ * Deliberately retire one named shared-app attempt using only exact durable
2557
+ * resource IDs. Unlike legacy destroy(), this never infers ownership from a
2558
+ * Machine name, metadata, sandboxId, a volume name, or an address listing.
2559
+ */
2560
+ async destroyAttempt(attemptId, signal) {
2561
+ if (!signal)
2562
+ throw new Error('Fly attempt destruction requires its lifecycle lock signal');
2563
+ if (this.appPerProject()) {
2564
+ throw new Error('Use explicit per-project app destruction for a per-project Fly attempt');
2565
+ }
2566
+ const journal = this.config.createAttemptJournal;
2567
+ if (!journal)
2568
+ throw new Error('Fly attempt destruction requires its durable journal');
2569
+ const snapshot = await journal.get(attemptId);
2570
+ if (signal.aborted)
2571
+ throw signal.reason;
2572
+ if (!snapshot)
2573
+ return;
2574
+ const forgotten = await this.retireCreateAttempt(snapshot, signal, {
2575
+ deleteApp: false,
2576
+ forget: true,
2577
+ recordedResourcesOnly: true,
2578
+ });
2579
+ if (!forgotten) {
2580
+ throw new Error(`Could not complete exact-resource cleanup for attempt "${attemptId}"; its intent was retained`);
2581
+ }
2582
+ }
1500
2583
  /**
1501
2584
  * Destroys a sandbox.
1502
2585
  *
@@ -1514,15 +2597,19 @@ class FlyioSandboxProvider {
1514
2597
  * other than a 404 (which counts as already gone), naming the failed step and
1515
2598
  * HTTP status, so the caller can retry instead of forgetting a billed app.
1516
2599
  */
1517
- async destroy(id) {
2600
+ async destroy(id, signal) {
2601
+ if (!signal)
2602
+ throw new Error('Fly destruction requires its lifecycle lock signal');
1518
2603
  const { app, machineId } = parseSandboxId(id);
1519
2604
  let volumeIds = [];
1520
2605
  let privateRoutes = {};
2606
+ let createAttemptId;
2607
+ let observedMachine = null;
1521
2608
  try {
1522
- const machine = await this.client.request(`/apps/${app}/machines/${machineId}`, {
1523
- nullOn: [404],
1524
- });
2609
+ const machine = await this.requestForDestruction(`/apps/${app}/machines/${machineId}`, { nullOn: [404] }, signal);
2610
+ observedMachine = machine;
1525
2611
  privateRoutes = decodePrivateRoutes(machine?.config?.metadata?.[`${this.metadataPrefix}.${PRIVATE_ROUTES_METADATA_SUFFIX}`]);
2612
+ createAttemptId = machine?.config?.metadata?.[`${this.metadataPrefix}.createAttempt`];
1526
2613
  if (!this.appPerProject()) {
1527
2614
  volumeIds = (machine?.config?.mounts ?? [])
1528
2615
  .map((mount) => mount.volume)
@@ -1530,19 +2617,60 @@ class FlyioSandboxProvider {
1530
2617
  }
1531
2618
  }
1532
2619
  catch (error) {
2620
+ if (signal.aborted)
2621
+ throw signal.reason;
1533
2622
  // Best-effort metadata gathering; the Machine may already be gone.
1534
2623
  // Destruction proceeds either way — but say so, because an unread
1535
2624
  // metadata block means the private addresses below are NOT released.
1536
2625
  logger.warn('Could not read the Fly Machine before destroy — its volumes and any private (Flycast) ' +
1537
2626
  'addresses it recorded will not be cleaned up', { id, error });
1538
2627
  }
2628
+ const journal = this.config.createAttemptJournal;
2629
+ if (journal) {
2630
+ const durable = await journal.forApp(app);
2631
+ if (signal.aborted)
2632
+ throw signal.reason;
2633
+ // Metadata is only a lookup hint. It selects an attempt when its durable
2634
+ // exact Machine identity also names the caller's ID; copied metadata must
2635
+ // never redirect retirement to a different Machine's attempt.
2636
+ let intent = durable.find((candidate) => candidate.attemptId === createAttemptId &&
2637
+ this.recordedMachineId(candidate) === machineId) ?? durable.find((candidate) => candidate.sandboxId === id);
2638
+ if (!intent && observedMachine) {
2639
+ intent = await this.journalLegacyMachine(app, observedMachine);
2640
+ }
2641
+ if (intent) {
2642
+ const current = await journal.get(intent.attemptId);
2643
+ if (signal.aborted)
2644
+ throw signal.reason;
2645
+ if (!current)
2646
+ throw new Error(`Fly create attempt "${intent.attemptId}" disappeared`);
2647
+ const durableMachineId = this.recordedMachineId(current);
2648
+ if (durableMachineId !== machineId) {
2649
+ throw new Error(`Refusing to retire attempt "${current.attemptId}": caller named Machine "${machineId}" but durable identity is "${durableMachineId ?? 'unresolved'}"`);
2650
+ }
2651
+ const forgotten = await this.retireCreateAttempt(current, signal, {
2652
+ deleteApp: this.appPerProject(),
2653
+ forget: true,
2654
+ observedMachines: observedMachine ? [observedMachine] : [],
2655
+ expectedAppName: app,
2656
+ expectedMachineId: machineId,
2657
+ });
2658
+ if (!forgotten) {
2659
+ throw new Error(`Could not complete every cleanup step for "${id}"; its intent was retained`);
2660
+ }
2661
+ return;
2662
+ }
2663
+ if (durable.length) {
2664
+ throw new Error(`Refusing to destroy "${id}" without its durable ownership record; ${durable.length} other attempt(s) remain`);
2665
+ }
2666
+ throw new Error(`Refusing to destroy "${id}" before its legacy ownership can be journaled`);
2667
+ }
1539
2668
  try {
1540
- await this.client.request(`/apps/${app}/machines/${machineId}?force=true`, {
1541
- method: 'DELETE',
1542
- nullOn: [404],
1543
- });
2669
+ await this.requestForDestruction(`/apps/${app}/machines/${machineId}?force=true`, { method: 'DELETE', nullOn: [404] }, signal);
1544
2670
  }
1545
2671
  catch (error) {
2672
+ if (signal.aborted)
2673
+ throw signal.reason;
1546
2674
  // The app delete is not attempted: removing the app under a Machine that
1547
2675
  // is still alive would hide it. The private addresses are left for the
1548
2676
  // retry, which re-reads them from the Machine's metadata.
@@ -1552,7 +2680,13 @@ class FlyioSandboxProvider {
1552
2680
  if (this.appPerProject()) {
1553
2681
  let appError;
1554
2682
  try {
1555
- await this.client.request(`/apps/${app}`, { method: 'DELETE', nullOn: [404] });
2683
+ const remaining = await this.readAttemptAppState(app, signal);
2684
+ if (remaining.machines.length) {
2685
+ throw new Error(`Refusing to delete Fly app "${app}" while a Machine remains`);
2686
+ }
2687
+ if (!remaining.appGone) {
2688
+ await this.requestForDestruction(`/apps/${app}`, { method: 'DELETE', nullOn: [404] }, signal);
2689
+ }
1556
2690
  }
1557
2691
  catch (error) {
1558
2692
  appError = error;
@@ -1563,24 +2697,41 @@ class FlyioSandboxProvider {
1563
2697
  }
1564
2698
  // Released even when the app delete failed: the Machine is gone, so a
1565
2699
  // retry could no longer read these addresses from its metadata.
1566
- await this.releasePrivateRoutes(privateRoutes);
2700
+ const routesReleased = await this.releasePrivateRoutes(privateRoutes, signal, this.networkFor(app));
1567
2701
  if (appError !== undefined) {
1568
2702
  throw new Error(describeDestroyFailure('app delete', id, appError), { cause: appError });
1569
2703
  }
2704
+ if (!routesReleased && createAttemptId && this.config.createAttemptJournal) {
2705
+ throw new Error(`Could not release every external Flycast address for "${id}"`);
2706
+ }
2707
+ if (signal.aborted)
2708
+ throw signal.reason;
2709
+ if (createAttemptId)
2710
+ await this.config.createAttemptJournal?.forget(createAttemptId);
1570
2711
  return;
1571
2712
  }
2713
+ let cleanupFailed = false;
1572
2714
  for (const volumeId of volumeIds) {
1573
2715
  try {
1574
- await this.client.request(`/apps/${app}/volumes/${volumeId}`, {
1575
- method: 'DELETE',
1576
- nullOn: [404],
1577
- });
2716
+ await this.requestForDestruction(`/apps/${app}/volumes/${volumeId}`, { method: 'DELETE', nullOn: [404] }, signal);
1578
2717
  }
1579
2718
  catch (error) {
2719
+ if (signal.aborted)
2720
+ throw signal.reason;
2721
+ cleanupFailed = true;
1580
2722
  logger.warn('Failed to remove Fly volume', { app, volumeId, error });
1581
2723
  }
1582
2724
  }
1583
- await this.releasePrivateRoutes(privateRoutes);
2725
+ cleanupFailed =
2726
+ !(await this.releasePrivateRoutes(privateRoutes, signal, this.networkFor(app))) ||
2727
+ cleanupFailed;
2728
+ if (cleanupFailed && createAttemptId && this.config.createAttemptJournal) {
2729
+ throw new Error(`Could not complete every cleanup step for "${id}"; its intent was retained`);
2730
+ }
2731
+ if (signal.aborted)
2732
+ throw signal.reason;
2733
+ if (createAttemptId)
2734
+ await this.config.createAttemptJournal?.forget(createAttemptId);
1584
2735
  }
1585
2736
  // ---------------------------------------------------------------------------
1586
2737
  // Templates
@@ -1659,19 +2810,15 @@ class FlyioSandboxProvider {
1659
2810
  return templates.removeTemplate(this.templateContext(), templateId);
1660
2811
  }
1661
2812
  /**
1662
- * Restores a template into a freshly-created Machine.
1663
- *
1664
- * A failure here DESTROYS the Machine. The alternative is to hand back a
1665
- * sandbox that started cleanly and whose filesystem is not the one that was
1666
- * asked for — the same silent-wrong-contents failure that makes a missing
1667
- * template a hard error rather than a fallback to the base image.
2813
+ * Restores a template into a freshly-created Machine. A failure is journaled
2814
+ * by create and settled by the reconciler, never by an inline delete.
1668
2815
  * @param app - The Fly app name.
1669
2816
  * @param machineId - The Fly Machine id.
1670
2817
  * @param templateId - The caller's template identifier.
1671
2818
  * @param manifest - The CONTROL-PLANE manifest, whose `capturePaths` bound what is extracted.
1672
- * @throws {Error} When the restore fails; the Machine is destroyed first.
2819
+ * @throws {Error} When the restore fails.
1673
2820
  */
1674
- async restoreTemplate(app, machineId, templateId, manifest) {
2821
+ async restoreTemplate(app, machineId, templateId, manifest, owner) {
1675
2822
  const ctx = this.templateContext();
1676
2823
  const store = templates.requireStore(ctx);
1677
2824
  const leaseId = `${machineId}-${Date.now().toString(36)}`.replace(/[^A-Za-z0-9_.-]/g, '_');
@@ -1679,10 +2826,10 @@ class FlyioSandboxProvider {
1679
2826
  try {
1680
2827
  // The restore runs through exec, so the Machine has to be up. `create`
1681
2828
  // launches it but returns as soon as Fly accepts the request.
1682
- await this.waitForStarted(app, machineId);
2829
+ await this.waitForStarted(app, machineId, owner);
1683
2830
  lease = await templates.acquireRestoreLease(ctx, templateId, leaseId);
1684
2831
  const url = await store.presignGet(templates.archiveKey(store, templateId), ctx.presignExpirySeconds);
1685
- const result = await ctx.exec(app, machineId, templates.buildRestoreCommand(manifest.capturePaths, url), ctx.transferTimeoutMs);
2832
+ const result = await execCommand(this.rawExec(app, machineId, owner), templates.buildRestoreCommand(manifest.capturePaths, url), { cwd: '/', timeout: ctx.transferTimeoutMs }, '/');
1686
2833
  if (result.exitCode !== 0) {
1687
2834
  throw new Error(t('codeSandbox.flyio.error.templateRestoreFailed', { templateId, exitCode: String(result.exitCode) }, {
1688
2835
  defaultValue: `Restoring template "${templateId}" into Fly Machine ${machineId} failed ` +
@@ -1690,19 +2837,6 @@ class FlyioSandboxProvider {
1690
2837
  }));
1691
2838
  }
1692
2839
  }
1693
- catch (error) {
1694
- try {
1695
- await this.client.request(`/apps/${app}/machines/${machineId}?force=true`, {
1696
- method: 'DELETE',
1697
- nullOn: [404],
1698
- });
1699
- }
1700
- catch (cleanupError) {
1701
- logger.warn('Failed to destroy the Fly Machine whose template restore failed — it will keep ' +
1702
- 'billing until it is reaped', { app, machineId, error: cleanupError });
1703
- }
1704
- throw error;
1705
- }
1706
2840
  finally {
1707
2841
  if (lease)
1708
2842
  await templates.releaseRestoreLease(ctx, lease);
@@ -1768,15 +2902,32 @@ class FlyioSandboxProvider {
1768
2902
  : DEFAULT_PROBE_TIMEOUT_MS;
1769
2903
  let app = null;
1770
2904
  let machineId = null;
2905
+ let ownsProbeApp = false;
1771
2906
  let policyPorts;
1772
2907
  try {
1773
2908
  policyPorts = this.egressAllowedPorts();
1774
- app = this.resolveApp(this.probeProjectId());
1775
- // The SAME policy a real sandbox gets (private-service ports included, so
1776
- // the probe observes the real posture), but NO Flycast allocation: this
1777
- // app is deleted seconds from now, and allocating would leave one
1778
- // unreleasable address per probe on every target app — every 15 minutes.
1779
- await this.ensureApp(app, { privateRoutes: false });
2909
+ // A random name collision is an adoption, never DELETE authority. Retry
2910
+ // fresh names in per-project mode; shared mode keeps its designated app.
2911
+ for (let attempt = 0; attempt < 5; attempt++) {
2912
+ app = this.resolveApp(this.probeProjectId());
2913
+ const ownership = { created: false };
2914
+ // The SAME policy a real sandbox gets (private-service ports included),
2915
+ // but NO Flycast allocation. Accepted app-create ownership is captured
2916
+ // before any later policy/Machine step can fail.
2917
+ await this.ensureApp(app, {
2918
+ privateRoutes: false,
2919
+ requireCreated: this.appPerProject(),
2920
+ ownership,
2921
+ });
2922
+ ownsProbeApp = ownership.created;
2923
+ if (!this.appPerProject() || ownsProbeApp)
2924
+ break;
2925
+ }
2926
+ if (this.appPerProject() && !ownsProbeApp) {
2927
+ throw new Error('Could not allocate a fresh Fly egress probe app name');
2928
+ }
2929
+ if (!app)
2930
+ throw new Error('Could not resolve a Fly egress probe app');
1780
2931
  machineId = await this.createProbeMachine(app);
1781
2932
  await this.waitForStarted(app, machineId);
1782
2933
  const seconds = Math.min(MAX_WAIT_SECONDS, Math.max(1, Math.ceil(timeoutMs / 1000) + PROBE_EXEC_HEADROOM_SECONDS));
@@ -1804,7 +2955,7 @@ class FlyioSandboxProvider {
1804
2955
  };
1805
2956
  }
1806
2957
  finally {
1807
- await this.destroyProbe(app, machineId);
2958
+ await this.destroyProbe(app, machineId, ownsProbeApp);
1808
2959
  }
1809
2960
  }
1810
2961
  /**
@@ -1865,15 +3016,17 @@ class FlyioSandboxProvider {
1865
3016
  return machine.id;
1866
3017
  }
1867
3018
  /**
1868
- * Removes the probe Machine and, in per-project mode, its throwaway app.
3019
+ * Removes the accepted probe Machine and, in per-project mode, only an app
3020
+ * whose create POST this probe itself had accepted. Adopted/unknown apps KEEP.
1869
3021
  *
1870
3022
  * Best-effort by design: a cleanup failure must not turn a completed
1871
3023
  * observation into an error, and `auto_destroy` plus the bounded init sleep
1872
3024
  * already reap the Machine on their own.
1873
3025
  * @param app - The probe app, or `null` when it was never resolved.
1874
- * @param machineId - The probe Machine id, or `null` when it was never created.
3026
+ * @param machineId - The accepted probe Machine id, or `null` when none was created.
3027
+ * @param appCreated - Accepted app-create provenance for this exact probe.
1875
3028
  */
1876
- async destroyProbe(app, machineId) {
3029
+ async destroyProbe(app, machineId, appCreated) {
1877
3030
  if (!app)
1878
3031
  return;
1879
3032
  if (machineId) {
@@ -1887,8 +3040,8 @@ class FlyioSandboxProvider {
1887
3040
  logger.warn('Failed to remove the Fly egress probe Machine', { app, machineId, error });
1888
3041
  }
1889
3042
  }
1890
- // Never in shared-app mode: that app holds real sandboxes.
1891
- if (!this.appPerProject())
3043
+ // Never in shared-app mode, nor for a colliding/adopted/unknown app.
3044
+ if (!this.appPerProject() || !appCreated)
1892
3045
  return;
1893
3046
  try {
1894
3047
  await this.client.request(`/apps/${app}`, { method: 'DELETE', nullOn: [404] });
@@ -1950,12 +3103,13 @@ class FlyioSandboxProvider {
1950
3103
  * @param machineId - The Fly Machine id.
1951
3104
  * @returns A transport callback for {@link execCommand}.
1952
3105
  */
1953
- rawExec(app, machineId) {
3106
+ rawExec(app, machineId, owner) {
1954
3107
  return async (command, timeoutSeconds) => {
1955
3108
  const deadline = Date.now() + EXEC_PRECONDITION_BUDGET_MS;
1956
3109
  for (let attempt = 1;; attempt++) {
1957
3110
  try {
1958
- const response = await this.client.request(`/apps/${app}/machines/${machineId}/exec`, {
3111
+ const path = `/apps/${app}/machines/${machineId}/exec`;
3112
+ const request = {
1959
3113
  method: 'POST',
1960
3114
  body: { command, timeout: timeoutSeconds },
1961
3115
  // The API's own timeout bounds the command; allow the HTTP request a
@@ -1965,7 +3119,10 @@ class FlyioSandboxProvider {
1965
3119
  // exec burst importFiles issues) is Fly API inconsistency under load,
1966
3120
  // not a real absence — retry it alongside the default 429/5xx.
1967
3121
  retryStatuses: [404],
1968
- });
3122
+ };
3123
+ const response = owner
3124
+ ? await this.requestForAttempt(owner, path, request)
3125
+ : await this.client.request(path, request);
1969
3126
  return response ?? {};
1970
3127
  }
1971
3128
  catch (error) {
@@ -2111,7 +3268,7 @@ class FlyioSandboxProvider {
2111
3268
  * @param machineId - The Fly Machine id.
2112
3269
  * @throws {Error} When the Machine has not started within the budget.
2113
3270
  */
2114
- async waitForStarted(app, machineId) {
3271
+ async waitForStarted(app, machineId, owner) {
2115
3272
  const totalSeconds = Math.max(1, this.config.startTimeoutSeconds ?? DEFAULT_START_TIMEOUT_SECONDS);
2116
3273
  const deadline = Date.now() + totalSeconds * 1000;
2117
3274
  let lastState;
@@ -2121,24 +3278,45 @@ class FlyioSandboxProvider {
2121
3278
  // enough: a host that does not have the image cached (every first boot
2122
3279
  // after an image push) pulls it first, which alone can exceed 60 s.
2123
3280
  for (;;) {
3281
+ if (owner) {
3282
+ await this.config.createAttemptJournal?.assertGeneration(owner.attemptId, owner.generation);
3283
+ if (owner.signal.aborted)
3284
+ throw owner.signal.reason;
3285
+ }
2124
3286
  const remaining = Math.ceil((deadline - Date.now()) / 1000);
2125
3287
  const seconds = Math.max(1, Math.min(remaining, MAX_WAIT_SECONDS));
2126
3288
  try {
2127
3289
  await this.client.request(`/apps/${app}/machines/${machineId}/wait?state=started&timeout=${seconds}`,
2128
3290
  // One attempt: a retry would re-block for the whole round, and the
2129
3291
  // state re-read below decides whether another round is warranted.
2130
- { attempts: 1, timeoutMs: (seconds + 15) * 1000 });
3292
+ { attempts: 1, timeoutMs: (seconds + 15) * 1000, signal: owner?.signal });
3293
+ if (owner) {
3294
+ await this.config.createAttemptJournal?.assertGeneration(owner.attemptId, owner.generation);
3295
+ if (owner.signal.aborted)
3296
+ throw owner.signal.reason;
3297
+ }
2131
3298
  return;
2132
3299
  }
2133
3300
  catch (error) {
3301
+ if (owner?.signal.aborted)
3302
+ throw owner.signal.reason;
2134
3303
  lastError = error;
2135
3304
  // The wait endpoint returning an error is not itself proof the Machine
2136
3305
  // failed — re-read the authoritative state before deciding.
2137
3306
  const machine = await this.client
2138
- .request(`/apps/${app}/machines/${machineId}`, { nullOn: [404] })
3307
+ .request(`/apps/${app}/machines/${machineId}`, {
3308
+ nullOn: [404],
3309
+ signal: owner?.signal,
3310
+ })
2139
3311
  .catch(() => null);
2140
- if (machine?.state === 'started')
3312
+ if (machine?.state === 'started') {
3313
+ if (owner) {
3314
+ await this.config.createAttemptJournal?.assertGeneration(owner.attemptId, owner.generation);
3315
+ if (owner.signal.aborted)
3316
+ throw owner.signal.reason;
3317
+ }
2141
3318
  return;
3319
+ }
2142
3320
  lastState = machine?.state;
2143
3321
  // Another round is useful only while the Machine is still on its way
2144
3322
  // up (`created` = provisioning/image pull, `starting` = boot). Any