@githolon/runtime 0.110.0 → 0.110.1

package/runner/node-entry.mjs

@@ -21835,6 +21835,15 @@ var ReplicaFollowController = class {
     }
     if (result.stillAhead || result.pushFailed || result.directOfferDeferred || result.reconnectRequired) {
       if (result.directOfferDeferred) {
+        if (!result.reconnectRequired) {
+          this.#desk.transition(result.workspace, {
+            case: ReplicaFollowEventKind.Failed,
+            nowUnixMillis,
+            failure: "remote workspace opening; retrying",
+            retryWithinMillis: 5e3
+          });
+          return delayUntilDue(this.#desk.get(result.workspace), nowUnixMillis);
+        }
         this.#desk.transition(result.workspace, {
           case: ReplicaFollowEventKind.DependencyDeferred,
           nowUnixMillis,
@@ -23740,6 +23749,34 @@ function resolveCustodyDisagreement(kernel, mount, requestId, policy, verdict) {
 init_define_NOMOS_KERNEL_RELEASE();
 var CUSTODY_OFFER_MEDIA_TYPE = "application/vnd.nomos.custody-offer-batch+protobuf";
 var CUSTODY_OFFER_RESULT_MEDIA_TYPE = "application/vnd.nomos.custody-offer-result+protobuf";
+function settleCustodyOfferExchange(awaiting, response) {
+  if (response.status === 202 && response.contentType?.includes("application/json")) {
+    try {
+      const pending = JSON.parse(new TextDecoder().decode(response.body));
+      if (pending?.retryable === true && typeof pending.preparing === "string") {
+        return { case: "maintenance", kind: pending.preparing };
+      }
+    } catch {
+    }
+  }
+  if (!response.contentType?.includes(CUSTODY_OFFER_RESULT_MEDIA_TYPE)) {
+    return {
+      case: "unexpected",
+      status: response.status,
+      ...response.contentType === void 0 ? {} : { contentType: response.contentType }
+    };
+  }
+  const verdict = decodeCustodyOffer(response.body, awaiting.requestId);
+  const receipt = custodyOfferReceipt(awaiting.requestId, verdict);
+  switch (verdict.case) {
+    case "admission":
+      return { case: "admission", value: verdict.value, receipt, status: response.status };
+    case "failure":
+      return { case: "failure", value: verdict.value, receipt, status: response.status };
+    case "blocked":
+      return { case: "blocked", value: verdict.value, receipt, status: response.status };
+  }
+}
 function encodeInlineCustodyOffer(args) {
   return toBinary(CustodyOfferBatchRequestSchema, create(CustodyOfferBatchRequestSchema, {
     protocolVersion: 2,
@@ -23866,6 +23903,7 @@ function summarizeCustodyAdmission(admission) {
 var PLACEMENT_REF3 = "refs/nomos/placement";
 var LAW_PRODUCTS_MEDIA_TYPE = "application/vnd.nomos.export-law-products-response+protobuf";
 var WorkspaceSyncPhase = Object.freeze({
+  Idle: "idle",
   Probe: "probe",
   Offer: "offer",
   Lease: "lease",
@@ -23899,7 +23937,6 @@ var WorkspaceSyncWork = class {
   #rediscover;
   /** The route the directory's answer yields, recomputed once that answer is observed. */
   #reroute;
-  #placementFetch;
   #discoveredFrontier;
   #observePlacement;
   /** Ask the directory for the provider's current placement frontier this round although a
@@ -23908,19 +23945,14 @@ var WorkspaceSyncWork = class {
   #directoryAnswered;
   #placementFetched;
   /** A round which converged without leasing, finishing once placement alone is fetched. */
-  #afterPlacement;
   #leased = false;
-  #lease;
   #command;
   #requestId;
   #started;
-  #phase = WorkspaceSyncPhase.Lease;
+  #state = { phase: WorkspaceSyncPhase.Idle };
   #localHead;
   #offerHead;
   #previousHead;
-  #fetchStarted;
-  #offerStarted;
-  #offerRequestId;
   #pushMillis = 0;
   #pushBytes = 0n;
   #desiredHead;
@@ -24015,7 +24047,7 @@ var WorkspaceSyncWork = class {
       throw new Error(`replica '${this.#binding.replicaId}' has no custody remote`);
     }
     if (this.#rediscover || this.#recheckPlacement || !this.#route?.placementObserved && !this.#route?.placementFrontier && !this.#kernel.resolveRef(this.#mount, this.#requestId, PLACEMENT_REF3)) {
-      this.#phase = WorkspaceSyncPhase.Discover;
+      this.#state = { phase: WorkspaceSyncPhase.Discover };
       return this.#remember({
         case: "pending",
         value: create(CorrespondencePendingSchema, {
@@ -24035,13 +24067,13 @@ var WorkspaceSyncWork = class {
   #openRound() {
     if (this.#receiveState || this.#replaceLocalHead !== void 0) return this.#leaseEffect();
     if (!this.#localHead) return this.#leaseEffect();
-    this.#offerRequestId = `${this.#binding.replicaId}:${this.#requestId}:${this.workId}:frontier`;
+    const requestId = `${this.#binding.replicaId}:${this.#requestId}:${this.workId}:frontier`;
     return this.#beginOffer(encodeInlineCustodyOffer({
       session: this.#binding.replicaId,
-      requestId: this.#offerRequestId,
+      requestId,
       observedHead: this.#localHead,
       intents: []
-    }), true);
+    }), { case: "awaiting", requestId }, true);
   }
   #offerOrReconcile() {
     if (this.#command.admit && this.#localHead && this.#localHead !== this.#previousHead) {
@@ -24062,13 +24094,13 @@ var WorkspaceSyncWork = class {
         throw new Error(`kernel did not classify transport for intent '${unspecified.intentId}'`);
       }
       if (suffix.length > 0) {
-        this.#offerStarted = this.#clock.monotonicNanoseconds();
-        this.#offerRequestId = `${this.#binding.replicaId}:${this.#requestId}:${this.workId}`;
+        const requestId2 = `${this.#binding.replicaId}:${this.#requestId}:${this.workId}`;
+        const startedAt = this.#clock.monotonicNanoseconds();
         this.#offerAfterConverge = true;
         this.#rebaseFrom = this.#previousHead;
         return this.#beginOffer(encodeInlineCustodyOffer({
           session: this.#binding.replicaId,
-          requestId: this.#offerRequestId,
+          requestId: requestId2,
           ...this.#previousHead === void 0 ? {} : { baseHead: this.#previousHead },
           observedHead: this.#localHead,
           intents: suffix,
@@ -24079,17 +24111,17 @@ var WorkspaceSyncWork = class {
             this.#previousHead
           ) : void 0,
           fastForwardOnly: true
-        }));
+        }), { case: "awaiting", requestId: requestId2, startedAt });
       }
     }
-    this.#offerRequestId = `${this.#binding.replicaId}:${this.#requestId}:${this.workId}:reconcile`;
+    const requestId = `${this.#binding.replicaId}:${this.#requestId}:${this.workId}:reconcile`;
     return this.#beginOffer(encodeInlineCustodyOffer({
       session: this.#binding.replicaId,
-      requestId: this.#offerRequestId,
+      requestId,
       baseHead: this.#previousHead,
       observedHead: this.#localHead,
       intents: []
-    }));
+    }), { case: "awaiting", requestId });
   }
   /**
    * NO COMMON BASE — a round that cannot converge, and must not pay for history to find out.
@@ -24165,18 +24197,18 @@ var WorkspaceSyncWork = class {
   #pullBeforeOffer() {
     this.#offerAfterConverge = true;
     this.#rebaseFrom = this.#previousHead;
-    this.#offerRequestId = `${this.#binding.replicaId}:${this.#requestId}:${this.workId}:pull-${this.#rebases}`;
+    const requestId = `${this.#binding.replicaId}:${this.#requestId}:${this.workId}:pull-${this.#rebases}`;
     return this.#beginOffer(encodeInlineCustodyOffer({
       session: this.#binding.replicaId,
-      requestId: this.#offerRequestId,
+      requestId,
       ...this.#previousHead === void 0 ? {} : { baseHead: this.#previousHead },
       observedHead: this.#localHead,
       intents: []
-    }));
+    }), { case: "awaiting", requestId });
   }
-  #leaseEffect() {
+  #leaseEffect(afterPlacement) {
     const token = this.#authToken();
-    this.#phase = WorkspaceSyncPhase.Lease;
+    this.#state = { phase: WorkspaceSyncPhase.Lease, ...afterPlacement ? { afterPlacement } : {} };
     return this.#remember({
       case: "pending",
       value: create(CorrespondencePendingSchema, {
@@ -24205,35 +24237,38 @@ var WorkspaceSyncWork = class {
         `workspace sync '${this.workId}' expected effect '${expected}', received '${receipt.effectId}'`
       );
     }
+    const state = this.#state;
     const previousReceipt = this.#lastReceiptEffectId;
     this.#lastReceiptEffectId = receipt.effectId;
     try {
       if (receipt.outcome.case === "failure") {
-        if (this.#phase === WorkspaceSyncPhase.Discover) return this.#openRound();
+        if (state.phase === WorkspaceSyncPhase.Discover) return this.#openRound();
         return this.#transportFailure(
           receipt.outcome.value.message,
           receipt.outcome.value.reason === PeerEffectFailureReason.OFFLINE
         );
       }
-      if (this.#phase === WorkspaceSyncPhase.Offer || this.#phase === WorkspaceSyncPhase.Probe) return this.#acceptOffer(receipt);
-      if (this.#phase === WorkspaceSyncPhase.Lease) return this.#acceptLease(receipt);
-      if (this.#phase === WorkspaceSyncPhase.Discover) return this.#acceptDiscovery(receipt);
-      if (this.#phase === WorkspaceSyncPhase.Placement) {
-        if (!this.#placementFetch || !this.#lease) throw new Error("placement transfer lost its lease");
-        this.#placementFetch.accept(receipt);
+      if (state.phase === WorkspaceSyncPhase.Offer || state.phase === WorkspaceSyncPhase.Probe) {
+        return this.#acceptOffer(receipt, state);
+      }
+      if (state.phase === WorkspaceSyncPhase.Lease) return this.#acceptLease(receipt, state);
+      if (state.phase === WorkspaceSyncPhase.Discover) return this.#acceptDiscovery(receipt);
+      if (state.phase === WorkspaceSyncPhase.Placement) {
+        state.transfer.accept(receipt);
         this.#placementFetched?.();
-        if (this.#afterPlacement) return this.#finish(this.#afterPlacement);
-        return this.#beginMainFetch(this.#lease);
+        if (state.afterPlacement) return this.#finish(state.afterPlacement);
+        return this.#beginMainFetch(state.lease);
       }
-      if (this.#phase === WorkspaceSyncPhase.Fetch) return this.#acceptFetch(receipt);
+      if (state.phase === WorkspaceSyncPhase.Fetch) return this.#acceptFetch(receipt, state);
       throw new Error(`workspace sync '${this.workId}' is already complete`);
     } catch (error) {
       this.#lastReceiptEffectId = previousReceipt;
       throw error;
     }
   }
-  #beginOffer(body, probe = false) {
-    this.#phase = probe ? WorkspaceSyncPhase.Probe : WorkspaceSyncPhase.Offer;
+  #beginOffer(body, exchange, probe = false) {
+    const phase = probe ? WorkspaceSyncPhase.Probe : WorkspaceSyncPhase.Offer;
+    this.#state = { phase, exchange };
     const token = this.#authToken();
     this.#pushBytes += BigInt(body.byteLength);
     return this.#remember({
@@ -24243,7 +24278,7 @@ var WorkspaceSyncWork = class {
         effect: {
           case: "http",
           value: create(PeerHttpEffectSchema, {
-            effectId: `${this.workId}:${this.#phase}`,
+            effectId: `${this.workId}:${phase}`,
             method: PeerHttpMethod.POST,
             url: this.#offerEndpoint(),
             contentType: CUSTODY_OFFER_MEDIA_TYPE,
@@ -24260,28 +24295,30 @@ var WorkspaceSyncWork = class {
       })
     });
   }
-  #acceptOffer(receipt) {
-    this.#pushMillis = this.#offerStarted === void 0 ? 0 : elapsedMillis(this.#clock, this.#offerStarted);
+  #acceptOffer(receipt, state) {
+    const exchange = state.exchange;
+    this.#pushMillis = exchange.startedAt === void 0 ? 0 : elapsedMillis(this.#clock, exchange.startedAt);
     if (receipt.outcome.case !== "response") {
       throw new Error("platform returned the wrong receipt for a custody offer");
     }
-    const response = receipt.outcome.value;
-    if (!response.contentType?.includes(CUSTODY_OFFER_RESULT_MEDIA_TYPE)) {
+    const outcome = settleCustodyOfferExchange(exchange, receipt.outcome.value);
+    if (outcome.case === "maintenance") {
+      return this.#finishOfferOutcome(true, false);
+    }
+    if (outcome.case === "unexpected") {
       this.#pushError = create(RuntimeTransportFailureSchema, {
         endpoint: this.#offerEndpoint(),
-        status: response.status,
+        status: outcome.status,
         refused: false,
-        message: `Sync received an unexpected server response (HTTP ${response.status}; ${response.contentType || "no content type"}). No custody receipt was returned.`
+        message: `Sync received an unexpected server response (HTTP ${outcome.status}; ${outcome.contentType || "no content type"}). No custody receipt was returned.`
       });
       return this.#finishOfferOutcome(false);
     }
-    if (!this.#offerRequestId) throw new Error("custody offer has no request identity");
-    const outcome = decodeCustodyOffer(response.body, this.#offerRequestId);
-    this.#offerReceipt = custodyOfferReceipt(this.#offerRequestId, outcome);
+    this.#offerReceipt = outcome.receipt;
     if (outcome.case === "failure") {
       this.#pushError = create(RuntimeTransportFailureSchema, {
         endpoint: this.#offerEndpoint(),
-        status: response.status,
+        status: outcome.status,
         refused: false,
         message: `${outcome.value.code}: ${outcome.value.message}`
       });
@@ -24290,12 +24327,12 @@ var WorkspaceSyncWork = class {
     if (outcome.case === "blocked") {
       return this.#finishOfferOutcome(true);
     }
-    if (response.status < 200 || response.status >= 300) {
+    if (outcome.status < 200 || outcome.status >= 300) {
       this.#pushError = create(RuntimeTransportFailureSchema, {
         endpoint: this.#offerEndpoint(),
-        status: response.status,
-        refused: response.status === 422,
-        message: `custody offer returned an admission verdict with HTTP ${response.status}`
+        status: outcome.status,
+        refused: outcome.status === 422,
+        message: `custody offer returned an admission verdict with HTTP ${outcome.status}`
       });
       return this.#finishOfferOutcome(false);
     }
@@ -24303,7 +24340,7 @@ var WorkspaceSyncWork = class {
     this.#desiredHead = admission.main || void 0;
     if (!this.#desiredHead) return this.#transportFailure("resident peer returned no admitted frontier", false);
     this.#candidateDurable = admission.sealedMain === this.#desiredHead;
-    if (this.#phase === WorkspaceSyncPhase.Probe && this.#desiredHead !== this.#localHead) {
+    if (state.phase === WorkspaceSyncPhase.Probe && this.#desiredHead !== this.#localHead) {
       if (!this.#establishBase(this.#desiredHead)) return this.#noCommonBaseResult();
       if (this.#alreadyJudged(this.#desiredHead)) {
         return this.#finish(create(SyncWorkspaceResponseSchema, {
@@ -24365,7 +24402,7 @@ var WorkspaceSyncWork = class {
     if (admission.recoveryRequired !== false) return this.#leaseEffect();
     return this.#transportFailure("resident peer supplied neither the missing delta nor a history gap", false);
   }
-  #acceptLease(receipt) {
+  #acceptLease(receipt, state) {
     if (receipt.outcome.case !== "response") {
       throw new Error("platform returned the wrong receipt for a Git read lease");
     }
@@ -24381,21 +24418,25 @@ var WorkspaceSyncWork = class {
     this.#leased = true;
     const frontier = this.#placementFrontier();
     if (frontier) {
-      this.#placementFetch = new PlacementFetch(
+      const transfer = new PlacementFetch(
         this.#kernel,
         this.#mount,
         this.#requestId,
         this.workId,
         frontier
       );
-      const pending = this.#placementFetch.start(lease);
+      const pending = transfer.start(lease);
       if (pending) {
-        this.#phase = WorkspaceSyncPhase.Placement;
-        this.#lease = lease;
+        this.#state = {
+          phase: WorkspaceSyncPhase.Placement,
+          transfer,
+          lease,
+          ...state.afterPlacement ? { afterPlacement: state.afterPlacement } : {}
+        };
         return this.#remember({ case: "pending", value: pending });
       }
     }
-    if (this.#afterPlacement) return this.#finish(this.#afterPlacement);
+    if (state.afterPlacement) return this.#finish(state.afterPlacement);
     return this.#beginMainFetch(lease);
   }
   /** The provider's placement frontier as this round knows it: what the directory named in this
@@ -24415,8 +24456,7 @@ var WorkspaceSyncWork = class {
     if (!new PlacementFetch(this.#kernel, this.#mount, this.#requestId, this.workId, frontier).wanted()) {
       return void 0;
     }
-    this.#afterPlacement = presented;
-    return this.#leaseEffect();
+    return this.#leaseEffect(presented);
   }
   #acceptDiscovery(receipt) {
     this.#directoryAnswered?.();
@@ -24440,9 +24480,11 @@ var WorkspaceSyncWork = class {
     return this.#replaceLocalHead !== void 0 || this.#localHead === void 0 && this.#previousHead === void 0;
   }
   #beginMainFetch(lease) {
-    this.#phase = WorkspaceSyncPhase.Fetch;
     const lawProducts = this.#lawProductsAlongside();
-    this.#fetchStarted = this.#clock.monotonicNanoseconds();
+    this.#state = {
+      phase: WorkspaceSyncPhase.Fetch,
+      startedAt: this.#clock.monotonicNanoseconds()
+    };
     const haves = this.#replaceLocalHead !== void 0 ? this.#repairMissingCommit ? [] : [this.#kernel.resolveRef(this.#mount, this.#requestId, deferredRemote)].filter(
       (head) => !!head
     ) : [...new Set([
@@ -24473,7 +24515,7 @@ var WorkspaceSyncWork = class {
       })
     });
   }
-  #acceptFetch(receipt) {
+  #acceptFetch(receipt, state) {
     if (receipt.outcome.case !== "gitFetch") {
       throw new Error("platform returned the wrong receipt for a Git object fetch");
     }
@@ -24483,7 +24525,7 @@ var WorkspaceSyncWork = class {
     if (fetched.pack.byteLength > 0) {
       this.#kernel.applyPack(this.#mount, this.#requestId, fetched.pack, fetched.head, fetched.shallow);
     }
-    const prefetchMillis = this.#fetchStarted === void 0 ? 0 : elapsedMillis(this.#clock, this.#fetchStarted);
+    const prefetchMillis = elapsedMillis(this.#clock, state.startedAt);
     const candidateHead = this.#receiveState ? this.#kernel.resolveStatePosition?.(this.#mount, this.#requestId, fetched.head, this.#receiveState) : fetched.head;
     if (!candidateHead) {
       return this.#finish(create(SyncWorkspaceResponseSchema, {
@@ -24834,11 +24876,11 @@ var WorkspaceSyncWork = class {
         pushBytes: this.#pushBytes,
         pullBytes: BigInt(receivedBytes),
         serverMillis: this.#admissionSummary?.serverMillis ?? 0,
-        offerInPush: this.#offerRequestId !== void 0
+        offerInPush: this.#pushBytes > 0n
       })
     });
   }
-  #finishOfferOutcome(blocked) {
+  #finishOfferOutcome(blocked, reconnectRequired = true) {
     return this.#finish(create(SyncWorkspaceResponseSchema, {
       workspace: this.#mount.workspace,
       previousHead: this.#previousHead,
@@ -24846,7 +24888,7 @@ var WorkspaceSyncWork = class {
       admitted: BigInt(this.#admissionSummary?.admittedIntentIds.length ?? 0),
       refused: BigInt((this.#admissionSummary?.refusedMessages.length ?? 0) + this.#parkedLocal),
       deferred: BigInt(this.#admissionSummary ? this.#admissionSummary.blockedMessages.length + this.#admissionSummary.deadLetteredMessages.length + this.#admissionSummary.quarantinedMessages.length + this.#admissionSummary.deferredMessages.length : blocked ? 1 : 0),
-      reconnectRequired: true,
+      reconnectRequired,
       stillAhead: this.#localHead !== this.#previousHead,
       directOfferDeferred: blocked,
       offerReceipt: this.#offerReceipt,
@@ -24860,15 +24902,16 @@ var WorkspaceSyncWork = class {
     }));
   }
   #transportFailure(message, offline, status = -1) {
-    const pushFailed = this.#phase === WorkspaceSyncPhase.Offer;
+    const state = this.#state;
+    const pushFailed = state.phase === WorkspaceSyncPhase.Offer;
     const failedEndpoint = this.#offerEndpoint();
     const failure = create(RuntimeTransportFailureSchema, {
       endpoint: pushFailed ? failedEndpoint : this.#binding.cloud ?? "",
       status,
       message
     });
-    if (pushFailed) {
-      this.#pushMillis = this.#offerStarted === void 0 ? 0 : elapsedMillis(this.#clock, this.#offerStarted);
+    if (state.phase === WorkspaceSyncPhase.Offer) {
+      this.#pushMillis = state.exchange.startedAt === void 0 ? 0 : elapsedMillis(this.#clock, state.exchange.startedAt);
       this.#pushError = failure;
     }
     return this.#finish(create(SyncWorkspaceResponseSchema, {
@@ -24885,7 +24928,7 @@ var WorkspaceSyncWork = class {
         pushMillis: this.#pushMillis,
         pushBytes: this.#pushBytes,
         prefetchMillis: elapsedMillis(this.#clock, this.#started),
-        offerInPush: this.#offerRequestId !== void 0
+        offerInPush: this.#pushBytes > 0n
       })
     }));
   }
@@ -24895,7 +24938,7 @@ var WorkspaceSyncWork = class {
     return step;
   }
   #finish(value) {
-    this.#phase = WorkspaceSyncPhase.Done;
+    this.#state = { phase: WorkspaceSyncPhase.Done };
     this.#current = void 0;
     return this.#remember({ case: "synced", value });
   }