@serve.zone/appstore 33.19.0 → 33.21.0

package/dist_catalog/catalog.envelope.json

@@ -5,9 +5,9 @@
       "major": 1,
       "minor": 0
     },
-    "revision": 33000019000000,
-    "publishedAt": "2026-10-23T01:01:33.000Z",
-    "sourceRevision": "d69a7b32bdfbabd33b87c4b73a5abe9782c4199a",
+    "revision": 33000021000000,
+    "publishedAt": "2026-11-08T06:06:22.000Z",
+    "sourceRevision": "7c0c673d02585e2a0edef620d3f1d49e80124e33",
     "apps": [
       {
         "info": {
@@ -60,7 +60,7 @@
             "Docs": "https://serve.zone"
           }
         },
-        "latestVersion": "1.30.7",
+        "latestVersion": "1.30.9",
         "versions": [
           {
             "version": "1.0.0",
@@ -10213,10 +10213,834 @@
             },
             "configDigest": "sha256:82044beb90090587212c0fa60ffbe11f322a9bf26ac6bcdd4b8d9ffcfb415b85"
           },
+          {
+            "version": "1.29.12",
+            "config": {
+              "image": "code.foss.global/serve.zone/cloudly@sha256:be728be0dc6b86f910684067c3bf8db3f07404586f6a85ca00f079de6d8cd64d",
+              "targetPorts": [
+                {
+                  "name": "web",
+                  "port": 3000,
+                  "protocol": "http",
+                  "default": true
+                }
+              ],
+              "envVars": [
+                {
+                  "key": "SERVEZONE_ENVIRONMENT",
+                  "value": "production",
+                  "description": "Cloudly runtime environment.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_PUBLIC_ORIGIN",
+                  "value": "https://${SERVICE_DOMAIN}",
+                  "description": "Canonical public Cloudly origin including protocol and optional public port.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_LISTENPORT",
+                  "value": "3000",
+                  "description": "Internal Cloudly HTTP port inside the container.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_SSLMODE",
+                  "value": "external",
+                  "description": "Use external TLS termination through Onebox or dcrouter.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_SECRET_KEYRING_PATH",
+                  "value": "/run/secrets/cloudly-secrets-v2-keyring.json",
+                  "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_SECRET_KEYRING_FILE",
+                  "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.",
+                  "required": true,
+                  "secret": true,
+                  "delivery": {
+                    "type": "file",
+                    "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json",
+                    "uid": 0,
+                    "gid": 0,
+                    "mode": 256
+                  }
+                },
+                {
+                  "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH",
+                  "value": "/run/secrets/cloudly-workloadinit-approval.json",
+                  "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE",
+                  "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.",
+                  "required": true,
+                  "secret": true,
+                  "delivery": {
+                    "type": "file",
+                    "targetPath": "/run/secrets/cloudly-workloadinit-approval.json",
+                    "uid": 0,
+                    "gid": 0,
+                    "mode": 256
+                  }
+                },
+                {
+                  "key": "MONGODB_URL",
+                  "value": "${MONGODB_URI}",
+                  "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "MONGODB_NAME",
+                  "value": "${MONGODB_DATABASE}",
+                  "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "MONGODB_USER",
+                  "value": "${MONGODB_USERNAME}",
+                  "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "MONGODB_PASS",
+                  "value": "${MONGODB_PASSWORD}",
+                  "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "S3_ENDPOINT",
+                  "value": "onebox-corestore",
+                  "description": "S3 endpoint host provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "S3_PORT",
+                  "value": "9000",
+                  "description": "S3 endpoint port provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "S3_USESSL",
+                  "value": "false",
+                  "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.",
+                  "required": true
+                },
+                {
+                  "key": "S3_BUCKET",
+                  "value": "${S3_BUCKET}",
+                  "description": "Cloudly's own S3 bucket, provisioned by Onebox Corestore: registry storage and kept workload logs.",
+                  "required": true
+                },
+                {
+                  "key": "S3_ACCESSKEY",
+                  "value": "${S3_ACCESS_KEY}",
+                  "description": "S3 access key provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "S3_SECRETKEY",
+                  "value": "${S3_SECRET_KEY}",
+                  "description": "S3 secret key provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "CLOUDLY_BACKUP_CRON",
+                  "value": "15 2 * * *",
+                  "description": "Schedule of Cloudly's backup-all-services task, which backs up every workload service's databases (not its object storage); Cloudly registers the task only while this is set. Daily at 02:15 UTC, clear of Onebox's backup-archive-prune at 03:00.",
+                  "required": false
+                },
+                {
+                  "key": "SERVEZONE_ADMINACCOUNT",
+                  "description": "Optional private first-start Cloudly operator in username:password form. Required by the fresh empty-instance workflow, never the reviewer credential. Supply only through sealed secret input; no default or automatic generation. Remove the bootstrap input with the Onebox owner command after first boot; revoking its persisted Cloudly password is a separate fresh-human credential action. Existing installations leave this unset.",
+                  "required": false,
+                  "secret": true,
+                  "delivery": {
+                    "type": "launcher-environment",
+                    "variableName": "SERVEZONE_ADMINACCOUNT",
+                    "uid": 0,
+                    "gid": 0,
+                    "mode": 256
+                  }
+                }
+              ],
+              "platformRequirements": {
+                "mongodb": true,
+                "s3": true
+              },
+              "backupExclusions": [
+                {
+                  "storageRequest": "legacy-s3",
+                  "keyPrefixes": [
+                    "runtime-logs/"
+                  ],
+                  "reason": "Workload logs; a restore brings back state, not past logs."
+                }
+              ],
+              "platformOidc": {
+                "redirectPath": "/auth/oidc/callback",
+                "roles": [
+                  {
+                    "id": "admin",
+                    "label": "Administrator",
+                    "description": "Full Cloudly administration for this hosted instance."
+                  },
+                  {
+                    "id": "user",
+                    "label": "User",
+                    "description": "Standard authenticated Cloudly access for this hosted instance."
+                  }
+                ],
+                "environmentVariables": {
+                  "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER",
+                  "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID",
+                  "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET",
+                  "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI",
+                  "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE"
+                },
+                "clientAuthenticationMethod": "client_secret_basic"
+              },
+              "minOneboxVersion": "33.8.0",
+              "changelog": "Move Cloudly to 33.28.0 on the config of 1.29.11 for a host that does not fulfil storage.database.backup-exclusions: 1.29.12 differs from 1.29.11 only in its image, its changelog and one added optional declaration. The image is the published multi-platform index be728be0dc6b86f910684067c3bf8db3f07404586f6a85ca00f079de6d8cd64d (linux/amd64 and linux/arm64), not a rebuild or a floating tag. The added declaration is the optional, sealed owner-supplied private first-start operator input SERVEZONE_ADMINACCOUNT, word for word as 1.30.9 declares it: secret, not required, no value and no generator, delivered in the launcher environment. Cloudly reads it only when its database holds no human user, so it changes nothing for an existing installation, whether or not that installation already carries the key from an earlier start; a fresh empty installation supplies it through sealed secret input or uses the parent initial-admin bootstrap. After first boot the Onebox owner removes the encrypted startup alias with service patch-env unsetSecrets and the normal replacement; that removal never revokes the persisted Cloudly password, which only an explicit operator action through Cloudly's fresh-human credential management (revokePassword) does. This remains the interim delivery option of AGL-1246; AGL-1179 under AGL-1178 owns the final Onebox broker and removes both bootstrap paths. 1.29.12 does not declare the runtime log ledger databaseBackupExclusions and does not require storage.database.backup-exclusions; it keeps requiring storage.object-storage.backup-exclusions, the runtime-logs/ exclusion, CLOUDLY_BACKUP_CRON 15 2 * * *, the environment, storage mappings, encrypted keyring and WorkloadInit approval file deliveries, platform OIDC, health check and the Onebox minimum 33.8.0 of 1.29.11. From 1.29.11 the upgrade takes Cloudly 33.24.1, 33.24.2, 33.24.3, 33.25.0, 33.25.1, 33.26.0, 33.27.0 and 33.28.0, which no 1.29.x template ran: canonical service hostnames at dcrouter and gateway refresh fixes (33.24.1); per-node fencing of node requests and the managed VPN owner (33.24.2); immediate replacement of runs whose network epoch ended and successor admission only after an accepted removal report (33.24.3); the writable dcrouterMailEndpointUrl setting, one shared per-node push bound and the cluster ingress run begun first (33.25.0); serialized task runs, cancelled task state kept, ingress certificate renewal, bounded node deliveries, fenced cluster and node changes, serialized CoreMail publication, resumed interrupted cleanup and no lifecycle writes from reads (33.25.1); the native Socket9 monitor at /typedrequest/native/v9 beside the unchanged Socket8 surfaces, whose full use needs a verified administrator, and the qualified 8.0.2 Node 24 runtime without package managers (33.26.0); a hosted Cloudly presents its parent's machine identity from SERVEZONE_APP_CONTROL_TOKEN, so on a Onebox it reports itself hosted and can show and start its parent-managed upgrade (33.27.0); dashboard release redeploys, the fleet Spark target in the database with per-node holds, previewed and confirmed node actions on node runtime nodes, a backup recorded failed unless it captured everything its service declares, a named refusal of runtime spec writes that would move service data, one route table per refresh without withdrawing routes that keep a usable certificate, and every in-place workload restore refused restore-operation-unavailable until a restore is an operation its node owns (33.28.0). An upgrade from 1.29.11 or 1.29.10 runs six forward-only data steps: service-hostname-canonical (0.33.0 to 0.34.0), runtime-authority-tokens (0.34.0 to 0.35.0), runtime-assignment-failures (0.35.0 to 0.36.0), bounded-slot-history (0.36.0 to 0.37.0), spark-fleet-target (0.37.0 to 0.38.0), which gives an installation that ran the code pin the Spark release 35.1.18 with its published hash as its fleet target, and node-action-carrier-expiry (0.38.0 to 0.39.0), which expires and keeps every open Swarm-era node action on a node without a Swarm-era session; an upgrade from 1.29.8 or earlier also runs retire-stale-reconciliation-stages (0.32.0 to 0.33.0) first. These steps have no way back: every earlier Cloudly image refuses data version 0.39.0, so stop Cloudly and verify a pre-upgrade database backup before the upgrade, and return to 1.29.11 or any earlier template only by restoring that backup, never by selecting an older image over the migrated database. Keep breaking, requiresManualReview, migrationRequired and backupBeforeUpgrade set. 1.29.12 sorts below 1.30.0, so it is never the upgrade target of a 1.30.x installation. 1.30.8 and 1.30.9 run the older Cloudly 33.26.0 image, which refuses data version 0.39.0: an installation on 1.29.12 must not move to 1.30.8 or 1.30.9, even on a host that later fulfils storage.database.backup-exclusions, and waits for a 1.30.x template on Cloudly 33.28.0 or later. A catalog publication does not install this version or upgrade a control plane. The rest of this changelog is that of 1.29.11, the Cloudly 33.24.0 template, and describes the changes of Cloudly 33.24.0 and the releases before it that an upgrade from 1.29.10 or earlier also takes: Move Cloudly to 33.24.0 on the config of 1.29.10: 1.29.11 differs from 1.29.10 only in its image and changelog. Cloudly 33.24.0 adds Temp tokens to the dashboard: Security, Temp tokens (/temp-tokens) mints, lists and revokes delegated admin temp tokens, so an administrator who signs in with a passkey through the platform provider (Onebox) mints one without a Cloudly password. A mint states its scopes (Secrets, Service credentials), its secret key prefixes (only with Secrets, at most 16) and a lifetime of 1 to 60 minutes. The token is shown once, in the answer's dialog, which drops it when it closes; the dashboard never writes it to a state part, browser storage, the URL or a log. Minting still needs a sign-in within the last five minutes, the rule Cloudly already applied and keeps unchanged: the page says until when the session can mint and offers Sign in again outside that window, and a provider session older than five minutes is signed into again at the provider first. The list shows the active tokens newest first, and Revoke asks for confirmation first. The release adds no data step, no environment variable, no setting, no collection and no index and changes neither the Dockerfile nor the qenv.yml requirements, so 1.29.11 keeps 1.29.10's runtime contract, CLOUDLY_BACKUP_CRON 15 2 * * * included, the data version 0.33.0 and the Onebox minimum 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and an upgrade from 1.29.8 or earlier runs the forward-only data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0, which has no way back: Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.29.8, 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. 1.29.11 does not declare the runtime log ledger databaseBackupExclusions; 1.30.7 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.11 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.7, and 1.29.11 is never the upgrade target of 1.30.0 to 1.30.6, which sort above it. Like 1.29.10, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.10 the upgrade moves Cloudly from 33.23.2 to 33.24.0 and runs no data step. From 1.29.9 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.10, the Cloudly 33.23.2 template, and describes the changes of Cloudly 33.23.2 and the releases before it that an upgrade from 1.29.9 or earlier also takes: Move Cloudly to 33.23.2, a fixes-only patch over 33.23.0, on the config of 1.29.9: 1.29.10 differs from 1.29.9 only in its image and changelog. Cloudly 33.23.2 takes the changes of Cloudly 33.23.1, which no template ran. Cloudly 33.23.1 composes each service's status on its own: a service whose stored image deployment plan, rollout status or newest deployment operation violates the @serve.zone/interfaces contract in a field the status reads gets the status failed with one composition stage that names the violation, for example: Cloudly cannot compose this service's status: its stored image deployment plan violates the contract (releaseTag, operationId missing). A composition that throws gives that service the same failed status with the reason and one error log line naming the service, and every other service keeps its status. Cloudly 33.23.0 answered the administrator's services list, getServiceById of such a service and the MCP tool cloudly_list_services with an internal error when one service stored a plan written before the plan carried releaseTag and operationId. The stored documents are not changed; a new promotion replaces such a plan. Cloudly 33.23.2 lets servezone service credential ensure give a service whose SERVEZONE_PLATFORM_AUTHORIZATION an operator stored as an ordinary secret its service machine credential: ensure, run by a human administrator who signed in within five minutes or with a serviceCredentials temp token, takes that operator-managed row over in the same transaction, active, retired, revoked or deleted, revokes every legacy version that could still be issued, keeps every version, moves the row to the credential's platform scope stamped with the administrator and the token, records the takeover on the new version, publishes the service's secret manifests again and is fenced on the operator authority and revision it read. Before, ensure was refused SERVICE_CREDENTIAL_CONFLICT for such a service whatever an administrator did first. A secret of the key under any other authority is still refused, and the legacy user token itself is not revoked automatically. Neither release adds a data step, an environment variable, a setting, a collection or an index, and neither changes the Dockerfile or the qenv.yml requirements, so 1.29.10 keeps 1.29.9's runtime contract, CLOUDLY_BACKUP_CRON 15 2 * * * included, the data version 0.33.0 and the Onebox minimum 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and an upgrade from 1.29.8 or earlier runs the forward-only data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0, which has no way back: Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.29.8, 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. 1.29.10 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.10 still exports the ledger with its documents; 1.30.6 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.10 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.6, and 1.29.10 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2, 1.30.3, 1.30.4 or 1.30.5, which sort above it. Like 1.29.9, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.9 the upgrade moves Cloudly from 33.23.0 to 33.23.2 and runs no data step. From 1.29.8 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.9, the Cloudly 33.23.0 template, and describes the changes of Cloudly 33.23.0 and the releases before it that an upgrade from 1.29.8 or earlier also takes: Move Cloudly to 33.23.0 on the config of 1.29.8: 1.29.9 differs from 1.29.8 only in its image and changelog. Cloudly 33.23.0 adds one forward-only data step, retire-stale-reconciliation-stages from 0.32.0 to 0.33.0, and takes the changes of Cloudly 33.22.0, which no template ran. A service's status is now what the producers Cloudly runs state about it: the Pallet runtime's verdict on each target, the current plan's rollout, the newest deployment operation and the records of the observers Cloudly runs (externalGateway, externalGatewayMail, coreMail and platformBinding). The Services list, the Overview and the MCP tool cloudly_list_services read that one status, so a service that showed Failed or Converging for good because of a stage the Swarm-era Coreflow reconciler last reported on 2026-08-03 shows its real state. getServices composes the status for an administrator only; a cluster's relay gets the services without it. The Overview lists every failed service with its reason and a link to it (degraded ones as warnings); its Deployments tile and Recent deployment changes table are gone, and so is the dashboard summary's deployments count. Cloudly no longer serves reportReconciliationStatus and reportDeploymentSnapshots, the Swarm-era Coreflow intake no component sends, and an observer deletes its record when what it observes goes: a route withdrawn from a service and a deleted or tombstoned platform binding withdraw their externalGateway and platformBinding record. The data step deletes, through the database driver before any model is prepared, every reconciliation record of a kind only the Swarm-era Coreflow reconciler wrote (workload, routing, immutable-image, immutable-adoption and immutable-attestation) and every externalGateway record of a route its service no longer declares; a record of a route still declared stays, failed or not, so do the records of other observers, and a rerun removes nothing. The deployment operations get the index service_deployment_operation_newest, which the model builds at start. Cloudly 33.23.0 adds delegated admin temp tokens: a human administrator who signed in within five minutes mints a token for the scopes secrets and serviceCredentials that lives one hour at most (servezone admin temp-token mint, list and revoke); Cloudly stores only its SHA-256, in cloudly_delegated_admin_tokens, records every use, refusal and failed write in cloudly_delegated_admin_token_audit, and a token never writes a reserved name and never deletes. Cloudly 33.23.0 moves to @serve.zone/interfaces 32.61.0 and its dashboard to @design.estate/dees-catalog 23 (controls are 30px tall instead of 32px, dialogs and sheets have 14px corners instead of 20px). With Cloudly 33.22.0 the scheduled backup-all-services task, which 1.29.8 turned on with CLOUDLY_BACKUP_CRON, backs up each workload service on the node and cluster its data lives on: on a Cloudly with more than one cluster every service was refused, and on one with a single cluster every backup was refused backup-node-required, while the run was recorded completed. A service with nothing to back up is skipped, a service without a target fails by name, and a run in which any service failed ends failed and names each one. Cloudly 33.22.0 adds servezone service credential inspect, ensure, rotate and revoke for the service machine credential @serve.zone/platformclient 4.0.0 and later signs in with, and names a refused actor SERVICE_CREDENTIAL_UNAUTHORIZED. Neither release adds an environment variable or a setting, and neither changes the Dockerfile or the qenv.yml requirements, so 1.29.9 keeps 1.29.8's runtime contract, CLOUDLY_BACKUP_CRON 15 2 * * * included, and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: the data step has no way back. Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.29.8, 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. 1.29.9 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.9 still exports the ledger with its documents; 1.30.5 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.9 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.5, and 1.29.9 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2, 1.30.3 or 1.30.4, which sort above it. Like 1.29.8, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.8 the upgrade moves Cloudly from 33.21.4 to 33.23.0 and runs the data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0. From 1.29.7 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.8, the Cloudly 33.21.4 template, and describes the changes of Cloudly 33.21.4 and the releases before it that an upgrade from 1.29.7 or earlier also takes: Move Cloudly to 33.21.4, a fixes-only patch over 33.21.3, on the config of 1.29.7, and turn on Cloudly's scheduled workload backups: 1.29.8 differs from 1.29.7 only in its image, its changelog and the environment variable CLOUDLY_BACKUP_CRON. Cloudly registers its backup-all-services task only when CLOUDLY_BACKUP_CRON is set, and no earlier template set it, so the databases of the workload services Cloudly runs were never backed up on a schedule. This template sets it to 15 2 * * *: every day at 02:15 UTC, clear of Onebox's backup-archive-prune at 03:00, Cloudly creates a backup of every workload service with backup-enabled resources, and it keeps the newest 24 settled backups of each service (CLOUDLY_BACKUP_KEEP_LAST, which the template does not set, defaults to 24) and deletes older ones with their stored archives. These backups cover the workload databases; a workload's object storage is not part of them (AGL-473). The variable is a plain declaration with no secret and no delivery and is not required. A host that upgrades a service gives a key the service does not have yet the template's value, so an upgraded Cloudly runs the schedule; an operator's stored value for the key wins over the template's. Cloudly 33.21.4 derives the owner of a launcher-environment secret file from the identity the workload runs as (AGL-459): every launcher delivery was published 0:0 and 0400, so a workload that runs as another user could not read its secrets and WorkloadInit exited with SECRET_SOURCE. The manifest publication now derives the owner per target cluster from the spec's runAs and the pinned image's config.User with @serve.zone/interfaces 32.60.0 deriveLauncherSecretDeliveryOwner, an owner that does not derive is refused launcher-secret-ownership-unknown, and the manifests 33.21.3 published are published again on upgrade. On @serve.zone/interfaces 32.60.0, external registry answers no longer carry the stored credential, Test connection proves the credentials through the Docker Registry v2 login, and a service update merges only the fields it states, so Edit and Configure Mail work again for a service with a runtime spec or an immutable image. Cloudly 33.21.4 also fixes the dashboard (AGL-387 to AGL-401: the status bar, the fonts, the dropdown presets, the view actions, the Secrets wizard and index, the Logs view and the App Store view), refuses a deployment without an existing service and node, a node action the node can never receive and a Jump code for a cluster without a relay, and bootstraps a node with the pinned Spark 35.1.18 release. Cloudly 33.21.4 adds no data step and keeps the data version 0.32.0, so an upgrade from 1.29.7 runs no data step; it adds no index, no collection and no setting, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is that of 1.29.7 with CLOUDLY_BACKUP_CRON and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and an upgrade from 1.29.6 or earlier runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0, which has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. 1.29.8 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.8 still exports the ledger with its documents; 1.30.4 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.8 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.4, and 1.29.8 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2 or 1.30.3, which sort above it. Like 1.29.7, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.7 the upgrade moves Cloudly from 33.21.3 to 33.21.4 and turns the scheduled workload backups on. From 1.29.6 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.7, the Cloudly 33.21.3 template, and describes the changes of Cloudly 33.21.3 and the releases before it that an upgrade from 1.29.6 or earlier also takes: Move Cloudly to 33.21.3, a patch over 33.21.2 that adds one forward-only data step, on the config of 1.29.6: 1.29.7 differs from 1.29.6 only in its image and changelog. A service migrated from Swarm reaches its object storage at its node's Corestore endpoint again. The Secrets v2 cutover turned the Docker-era Corestore binding environment, the nine object-storage endpoint names and, for a binding with retention, the eleven S3_RETENTION_* names, into service-owned secrets in scope platform:cloudly-corestore, the endpoint names with the Docker host corestore; a Corestore binding now gives the endpoint names as plain environment composed from the node's endpoint, and the secrets the cutover left behind replaced that environment in the workload, since a secret wins over an inherited variable of the same name, so such a service exited at startup with getaddrinfo ENOTFOUND corestore. Cloudly 33.21.3 adds the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. It retires every active service-owned secret in that scope whose key is one of the twenty non-secret object-storage names (the nine endpoint names and the eleven S3_RETENTION_* names) as a lifecycle change retires one: the version is retired and kept, never purged, and the metadata names the step as the author. The service's secrets revision advances once, and its desired manifests are published again without those secrets under a new secret rollout, the previous one accepted for five more minutes, so the runtime replaces the service's replicas once; a single-replica service restarts briefly. A service whose image plan or secret deployment plan the runtime already refuses gets its manifest from its next promotion. The six credential names, other scopes, other owners and greenfield services are left alone; a database binding gives no plain name, so no database secret is touched. Each service is one transaction, so a rerun writes nothing. The step has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. Cloudly 33.21.3 adds no index, no collection, no setting and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.6 and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and every upgrade to this template runs the step to 0.32.0. 1.29.7 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.7 still exports the ledger with its documents; 1.30.3 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.7 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.3, and 1.29.7 is never the upgrade target of 1.30.0, 1.30.1 or 1.30.2, which sort above it. Like 1.29.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.6 the upgrade moves Cloudly from 33.21.2 to 33.21.3 and runs the step to 0.32.0. From 1.29.5 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.6, the Cloudly 33.21.2 template, and describes the changes of Cloudly 33.21.2 and the releases before it that an upgrade from 1.29.5 or earlier also takes: Move Cloudly to 33.21.2, a fixes-only patch over 33.21.1, on the config of 1.29.5: 1.29.6 differs from 1.29.5 only in its image and changelog. A recovery redeploy over a dead retained rollout plan, one that no runtime can execute (such as a Swarm-era plan) and whose operation is settled, is no longer refused PLATFORM_CAPABILITY_UNAVAILABLE by its reserve and promote preflights while its Corestore bindings wait for their credentials: such a service has no plan a runtime can run, so it is issued no credential grant until a promotion writes one, and its database and object-storage bindings could not become ready before the promotion those preflights refused. The reserve and promote preflights now defer the readiness of such a binding while it is requested or provisioning, as the promote preflight of a held service's first promotion already did, and warn PLATFORM_BINDING_READINESS_DEFERRED; a binding that adopts a legacy resource is deferred only while Corestore lists that resource as authorized for it, and the route preflight after the rollout still requires every binding ready. The retained plan is judged by the operation it names, as the promotion fence reads it, never by the service's latest operation. 1.29.6 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.6 still exports the ledger with its documents; 1.30.2 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.6 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.2, and 1.29.6 is never the upgrade target of 1.30.0 or 1.30.1, which sort above it. Cloudly 33.21.2 adds no data step, no index and no setting and keeps the data version 0.31.0, so an upgrade from 1.29.5, 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.5 and requires Onebox 33.8.0. Like 1.29.5, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.5 the upgrade moves Cloudly from 33.21.1 to 33.21.2. From 1.29.4 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5, the Cloudly 33.21.1 template, and describes the changes of Cloudly 33.21.1 and the releases before it that an upgrade from 1.29.4 or earlier also takes: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
+              "breaking": true,
+              "requiresManualReview": true,
+              "migrationRequired": true,
+              "backupBeforeUpgrade": true,
+              "requiresFeatures": [
+                "storage.object-storage.backup-exclusions"
+              ],
+              "healthCheck": {
+                "path": "/status",
+                "port": 3000,
+                "expectedStatus": 200
+              }
+            },
+            "configDigest": "sha256:8a5b44d0a4d7330ea71193c58d8fa3125c1402cf074a1aa28015d163428ccdb8"
+          },
+          {
+            "version": "1.29.14",
+            "config": {
+              "image": "code.foss.global/serve.zone/cloudly@sha256:50a2fab86d151197b468b9174d192f12de0bb58c7ad7ab66b9e8a930eb8c3a56",
+              "targetPorts": [
+                {
+                  "name": "web",
+                  "port": 3000,
+                  "protocol": "http",
+                  "default": true
+                }
+              ],
+              "envVars": [
+                {
+                  "key": "SERVEZONE_ENVIRONMENT",
+                  "value": "production",
+                  "description": "Cloudly runtime environment.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_PUBLIC_ORIGIN",
+                  "value": "https://${SERVICE_DOMAIN}",
+                  "description": "Canonical public Cloudly origin including protocol and optional public port.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_LISTENPORT",
+                  "value": "3000",
+                  "description": "Internal Cloudly HTTP port inside the container.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_SSLMODE",
+                  "value": "external",
+                  "description": "Use external TLS termination through Onebox or dcrouter.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_SECRET_KEYRING_PATH",
+                  "value": "/run/secrets/cloudly-secrets-v2-keyring.json",
+                  "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_SECRET_KEYRING_FILE",
+                  "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.",
+                  "required": true,
+                  "secret": true,
+                  "delivery": {
+                    "type": "file",
+                    "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json",
+                    "uid": 0,
+                    "gid": 0,
+                    "mode": 256
+                  }
+                },
+                {
+                  "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH",
+                  "value": "/run/secrets/cloudly-workloadinit-approval.json",
+                  "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE",
+                  "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.",
+                  "required": true,
+                  "secret": true,
+                  "delivery": {
+                    "type": "file",
+                    "targetPath": "/run/secrets/cloudly-workloadinit-approval.json",
+                    "uid": 0,
+                    "gid": 0,
+                    "mode": 256
+                  }
+                },
+                {
+                  "key": "MONGODB_URL",
+                  "value": "${MONGODB_URI}",
+                  "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "MONGODB_NAME",
+                  "value": "${MONGODB_DATABASE}",
+                  "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "MONGODB_USER",
+                  "value": "${MONGODB_USERNAME}",
+                  "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "MONGODB_PASS",
+                  "value": "${MONGODB_PASSWORD}",
+                  "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "S3_ENDPOINT",
+                  "value": "onebox-corestore",
+                  "description": "S3 endpoint host provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "S3_PORT",
+                  "value": "9000",
+                  "description": "S3 endpoint port provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "S3_USESSL",
+                  "value": "false",
+                  "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.",
+                  "required": true
+                },
+                {
+                  "key": "S3_BUCKET",
+                  "value": "${S3_BUCKET}",
+                  "description": "Cloudly's own S3 bucket, provisioned by Onebox Corestore: registry storage and kept workload logs.",
+                  "required": true
+                },
+                {
+                  "key": "S3_ACCESSKEY",
+                  "value": "${S3_ACCESS_KEY}",
+                  "description": "S3 access key provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "S3_SECRETKEY",
+                  "value": "${S3_SECRET_KEY}",
+                  "description": "S3 secret key provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "CLOUDLY_BACKUP_CRON",
+                  "value": "15 2 * * *",
+                  "description": "Schedule of Cloudly's backup-all-services task, which backs up every workload service's databases (not its object storage); Cloudly registers the task only while this is set. Daily at 02:15 UTC, clear of Onebox's backup-archive-prune at 03:00.",
+                  "required": false
+                },
+                {
+                  "key": "SERVEZONE_ADMINACCOUNT",
+                  "description": "Optional private first-start Cloudly operator in username:password form. Required by the fresh empty-instance workflow, never the reviewer credential. Supply only through sealed secret input; no default or automatic generation. Remove the bootstrap input with the Onebox owner command after first boot; revoking its persisted Cloudly password is a separate fresh-human credential action. Existing installations leave this unset.",
+                  "required": false,
+                  "secret": true,
+                  "delivery": {
+                    "type": "launcher-environment",
+                    "variableName": "SERVEZONE_ADMINACCOUNT",
+                    "uid": 0,
+                    "gid": 0,
+                    "mode": 256
+                  }
+                }
+              ],
+              "platformRequirements": {
+                "mongodb": true,
+                "s3": true
+              },
+              "backupExclusions": [
+                {
+                  "storageRequest": "legacy-s3",
+                  "keyPrefixes": [
+                    "runtime-logs/"
+                  ],
+                  "reason": "Workload logs; a restore brings back state, not past logs."
+                }
+              ],
+              "platformOidc": {
+                "redirectPath": "/auth/oidc/callback",
+                "roles": [
+                  {
+                    "id": "admin",
+                    "label": "Administrator",
+                    "description": "Full Cloudly administration for this hosted instance."
+                  },
+                  {
+                    "id": "user",
+                    "label": "User",
+                    "description": "Standard authenticated Cloudly access for this hosted instance."
+                  }
+                ],
+                "environmentVariables": {
+                  "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER",
+                  "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID",
+                  "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET",
+                  "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI",
+                  "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE"
+                },
+                "clientAuthenticationMethod": "client_secret_basic"
+              },
+              "minOneboxVersion": "33.8.0",
+              "changelog": "Move Cloudly to 33.28.1, a fixes-only patch over 33.28.0, on the config of 1.29.12: 1.29.14 differs from 1.29.12 only in its image and its changelog. The image is the published multi-platform index 50a2fab86d151197b468b9174d192f12de0bb58c7ad7ab66b9e8a930eb8c3a56 (linux/amd64 and linux/arm64), not a rebuild or a floating tag. Cloudly 33.28.1 keeps a site's certificate for as long as a service on the cluster declares its hostname, not only while its route is in the route table: a route that drops out of a reading for a moment, for example while its only replica is replaced, keeps its certificate and is served again at once when it returns, instead of waiting for a new certificate fetch, which kept the site unreachable for minutes, and for 15 minutes after a failed fetch. A renewal that replaces an expired held certificate now also refreshes the route table, so the renewed certificate is served without waiting for the next change. 33.28.1 adds no data step, no environment variable, no setting, no collection and no index and keeps data version 0.39.0. Upgrade path, from every template it is offered to (1.28.3, 1.28.5, 1.28.6 and 1.29.0 to 1.29.12, on a host that fulfils storage.object-storage.backup-exclusions): from 1.29.12 the upgrade changes only the image and runs no data step, so a return to 1.29.12 needs no restore. From 1.29.11 it takes Cloudly 33.24.1 to 33.28.1, from 1.29.10 also 33.24.0 and from 1.29.9 also 33.23.2, and each of the three runs six forward-only data steps: service-hostname-canonical (0.33.0 to 0.34.0), runtime-authority-tokens (0.34.0 to 0.35.0), runtime-assignment-failures (0.35.0 to 0.36.0), bounded-slot-history (0.36.0 to 0.37.0), spark-fleet-target (0.37.0 to 0.38.0) and node-action-carrier-expiry (0.38.0 to 0.39.0). From 1.29.8 or 1.29.7 it first also runs retire-stale-reconciliation-stages (0.32.0 to 0.33.0); from 1.29.6 to 1.29.2 also retire-corestore-environment-secrets (0.31.0 to 0.32.0); from 1.29.1, 1.29.0, 1.28.6, 1.28.5 or 1.28.3 also service-reconciliation-observations (0.30.0 to 0.31.0). These steps have no way back: every Cloudly image before 33.28.0 refuses data version 0.39.0, so stop Cloudly and verify a pre-upgrade database backup before an upgrade from 1.29.11 or earlier, and return to such a template only by restoring that backup, never by selecting an older image over the migrated database. Keep breaking, requiresManualReview, migrationRequired and backupBeforeUpgrade set. 1.29.14 sorts below 1.30.0, so a 1.30.x installation never takes it. 1.30.8 and 1.30.9 run the older Cloudly 33.26.0 image, which refuses data version 0.39.0: an installation on 1.29.14 must not move to 1.30.8 or 1.30.9, even on a host that later fulfils storage.database.backup-exclusions, and waits for a 1.30.x template on Cloudly 33.28.1 or later. 1.29.13 was a locally signed rollback template that was never published; its version is never used again, which is why this template is 1.29.14. A catalog publication does not install this version or upgrade a control plane. cloudly@1.29.12 holds the older cumulative history.",
+              "breaking": true,
+              "requiresManualReview": true,
+              "migrationRequired": true,
+              "backupBeforeUpgrade": true,
+              "requiresFeatures": [
+                "storage.object-storage.backup-exclusions"
+              ],
+              "healthCheck": {
+                "path": "/status",
+                "port": 3000,
+                "expectedStatus": 200
+              }
+            },
+            "configDigest": "sha256:d9fb784d57eacd729961db28a45d9667585e72651cf46bc5906d430b7a74e1d7"
+          },
           {
             "version": "1.30.0",
             "config": {
-              "image": "code.foss.global/serve.zone/cloudly@sha256:cd0bfc82575d9a4a9d0808e19ec61b0bf188d83729d66195afbf5f373164f16e",
+              "image": "code.foss.global/serve.zone/cloudly@sha256:cd0bfc82575d9a4a9d0808e19ec61b0bf188d83729d66195afbf5f373164f16e",
+              "targetPorts": [
+                {
+                  "name": "web",
+                  "port": 3000,
+                  "protocol": "http",
+                  "default": true
+                }
+              ],
+              "envVars": [
+                {
+                  "key": "SERVEZONE_ENVIRONMENT",
+                  "value": "production",
+                  "description": "Cloudly runtime environment.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_PUBLIC_ORIGIN",
+                  "value": "https://${SERVICE_DOMAIN}",
+                  "description": "Canonical public Cloudly origin including protocol and optional public port.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_LISTENPORT",
+                  "value": "3000",
+                  "description": "Internal Cloudly HTTP port inside the container.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_SSLMODE",
+                  "value": "external",
+                  "description": "Use external TLS termination through Onebox or dcrouter.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_SECRET_KEYRING_PATH",
+                  "value": "/run/secrets/cloudly-secrets-v2-keyring.json",
+                  "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_SECRET_KEYRING_FILE",
+                  "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.",
+                  "required": true,
+                  "secret": true,
+                  "delivery": {
+                    "type": "file",
+                    "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json",
+                    "uid": 0,
+                    "gid": 0,
+                    "mode": 256
+                  }
+                },
+                {
+                  "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH",
+                  "value": "/run/secrets/cloudly-workloadinit-approval.json",
+                  "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE",
+                  "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.",
+                  "required": true,
+                  "secret": true,
+                  "delivery": {
+                    "type": "file",
+                    "targetPath": "/run/secrets/cloudly-workloadinit-approval.json",
+                    "uid": 0,
+                    "gid": 0,
+                    "mode": 256
+                  }
+                },
+                {
+                  "key": "MONGODB_URL",
+                  "value": "${MONGODB_URI}",
+                  "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "MONGODB_NAME",
+                  "value": "${MONGODB_DATABASE}",
+                  "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "MONGODB_USER",
+                  "value": "${MONGODB_USERNAME}",
+                  "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "MONGODB_PASS",
+                  "value": "${MONGODB_PASSWORD}",
+                  "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "S3_ENDPOINT",
+                  "value": "onebox-corestore",
+                  "description": "S3 endpoint host provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "S3_PORT",
+                  "value": "9000",
+                  "description": "S3 endpoint port provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "S3_USESSL",
+                  "value": "false",
+                  "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.",
+                  "required": true
+                },
+                {
+                  "key": "S3_BUCKET",
+                  "value": "${S3_BUCKET}",
+                  "description": "Cloudly's own S3 bucket, provisioned by Onebox Corestore: registry storage and kept workload logs.",
+                  "required": true
+                },
+                {
+                  "key": "S3_ACCESSKEY",
+                  "value": "${S3_ACCESS_KEY}",
+                  "description": "S3 access key provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "S3_SECRETKEY",
+                  "value": "${S3_SECRET_KEY}",
+                  "description": "S3 secret key provisioned by Onebox Corestore.",
+                  "required": true
+                }
+              ],
+              "platformRequirements": {
+                "mongodb": true,
+                "s3": true
+              },
+              "backupExclusions": [
+                {
+                  "storageRequest": "legacy-s3",
+                  "keyPrefixes": [
+                    "runtime-logs/"
+                  ],
+                  "reason": "Workload logs; a restore brings back state, not past logs."
+                }
+              ],
+              "databaseBackupExclusions": [
+                {
+                  "database": "mongodb",
+                  "collections": [
+                    "cloudly_runtime_log_attempts",
+                    "cloudly_runtime_log_batches",
+                    "cloudly_runtime_log_captures",
+                    "cloudly_runtime_log_inline_payloads",
+                    "cloudly_runtime_log_node_discards",
+                    "cloudly_runtime_log_queue"
+                  ],
+                  "reason": "Workload log ledger; a restore brings back state, not past logs."
+                }
+              ],
+              "platformOidc": {
+                "redirectPath": "/auth/oidc/callback",
+                "roles": [
+                  {
+                    "id": "admin",
+                    "label": "Administrator",
+                    "description": "Full Cloudly administration for this hosted instance."
+                  },
+                  {
+                    "id": "user",
+                    "label": "User",
+                    "description": "Standard authenticated Cloudly access for this hosted instance."
+                  }
+                ],
+                "environmentVariables": {
+                  "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER",
+                  "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID",
+                  "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET",
+                  "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI",
+                  "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE"
+                },
+                "clientAuthenticationMethod": "client_secret_basic"
+              },
+              "minOneboxVersion": "33.8.0",
+              "changelog": "Move Cloudly to 33.21.0, a feature release over 33.20.4: Cloudly's runtime log ledger is no longer part of its backups, and a restore keeps the logs of every running workload. This template declares the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, as databaseBackupExclusions of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. The ledger holds most of a long-running Cloudly's database, so its backups shrink to the state they protect. Every Cloudly 33.21.0 start writes the log record of each running attempt that has none, before it serves node requests, so the nodes' next batches are kept again instead of being refused as attempts whose logs Cloudly does not keep; the first batch of each capture after a restore states the numbers the restore lost as a gap. A start whose attempts all have their records writes nothing, and an attempt whose record cannot be written is named in a warning and tried again at the next start. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.4; a host without the object-storage feature keeps 1.28.4. Like 1.29.4, this template declares Cloudly's workload log objects non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3). Cloudly 33.21.0 adds no data step and no index and keeps the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection, and the template requires Onebox 33.8.0. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.0. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
+              "breaking": true,
+              "requiresManualReview": true,
+              "migrationRequired": true,
+              "backupBeforeUpgrade": true,
+              "requiresFeatures": [
+                "storage.database.backup-exclusions",
+                "storage.object-storage.backup-exclusions"
+              ],
+              "healthCheck": {
+                "path": "/status",
+                "port": 3000,
+                "expectedStatus": 200
+              }
+            },
+            "configDigest": "sha256:e97c235d8f27d9210d9828a83a61035a7b42b11cba667826f3c7e3dfb37b813c"
+          },
+          {
+            "version": "1.30.1",
+            "config": {
+              "image": "code.foss.global/serve.zone/cloudly@sha256:1e92473c17522967711e8a82459fd8933b8c53b6bad26d0dba516cb9b1eccb34",
+              "targetPorts": [
+                {
+                  "name": "web",
+                  "port": 3000,
+                  "protocol": "http",
+                  "default": true
+                }
+              ],
+              "envVars": [
+                {
+                  "key": "SERVEZONE_ENVIRONMENT",
+                  "value": "production",
+                  "description": "Cloudly runtime environment.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_PUBLIC_ORIGIN",
+                  "value": "https://${SERVICE_DOMAIN}",
+                  "description": "Canonical public Cloudly origin including protocol and optional public port.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_LISTENPORT",
+                  "value": "3000",
+                  "description": "Internal Cloudly HTTP port inside the container.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_SSLMODE",
+                  "value": "external",
+                  "description": "Use external TLS termination through Onebox or dcrouter.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_SECRET_KEYRING_PATH",
+                  "value": "/run/secrets/cloudly-secrets-v2-keyring.json",
+                  "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_SECRET_KEYRING_FILE",
+                  "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.",
+                  "required": true,
+                  "secret": true,
+                  "delivery": {
+                    "type": "file",
+                    "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json",
+                    "uid": 0,
+                    "gid": 0,
+                    "mode": 256
+                  }
+                },
+                {
+                  "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH",
+                  "value": "/run/secrets/cloudly-workloadinit-approval.json",
+                  "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.",
+                  "required": true
+                },
+                {
+                  "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE",
+                  "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.",
+                  "required": true,
+                  "secret": true,
+                  "delivery": {
+                    "type": "file",
+                    "targetPath": "/run/secrets/cloudly-workloadinit-approval.json",
+                    "uid": 0,
+                    "gid": 0,
+                    "mode": 256
+                  }
+                },
+                {
+                  "key": "MONGODB_URL",
+                  "value": "${MONGODB_URI}",
+                  "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "MONGODB_NAME",
+                  "value": "${MONGODB_DATABASE}",
+                  "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "MONGODB_USER",
+                  "value": "${MONGODB_USERNAME}",
+                  "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "MONGODB_PASS",
+                  "value": "${MONGODB_PASSWORD}",
+                  "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "S3_ENDPOINT",
+                  "value": "onebox-corestore",
+                  "description": "S3 endpoint host provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "S3_PORT",
+                  "value": "9000",
+                  "description": "S3 endpoint port provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "S3_USESSL",
+                  "value": "false",
+                  "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.",
+                  "required": true
+                },
+                {
+                  "key": "S3_BUCKET",
+                  "value": "${S3_BUCKET}",
+                  "description": "Cloudly's own S3 bucket, provisioned by Onebox Corestore: registry storage and kept workload logs.",
+                  "required": true
+                },
+                {
+                  "key": "S3_ACCESSKEY",
+                  "value": "${S3_ACCESS_KEY}",
+                  "description": "S3 access key provisioned by Onebox Corestore.",
+                  "required": true
+                },
+                {
+                  "key": "S3_SECRETKEY",
+                  "value": "${S3_SECRET_KEY}",
+                  "description": "S3 secret key provisioned by Onebox Corestore.",
+                  "required": true
+                }
+              ],
+              "platformRequirements": {
+                "mongodb": true,
+                "s3": true
+              },
+              "backupExclusions": [
+                {
+                  "storageRequest": "legacy-s3",
+                  "keyPrefixes": [
+                    "runtime-logs/"
+                  ],
+                  "reason": "Workload logs; a restore brings back state, not past logs."
+                }
+              ],
+              "databaseBackupExclusions": [
+                {
+                  "database": "mongodb",
+                  "collections": [
+                    "cloudly_runtime_log_attempts",
+                    "cloudly_runtime_log_batches",
+                    "cloudly_runtime_log_captures",
+                    "cloudly_runtime_log_inline_payloads",
+                    "cloudly_runtime_log_node_discards",
+                    "cloudly_runtime_log_queue"
+                  ],
+                  "reason": "Workload log ledger; a restore brings back state, not past logs."
+                }
+              ],
+              "platformOidc": {
+                "redirectPath": "/auth/oidc/callback",
+                "roles": [
+                  {
+                    "id": "admin",
+                    "label": "Administrator",
+                    "description": "Full Cloudly administration for this hosted instance."
+                  },
+                  {
+                    "id": "user",
+                    "label": "User",
+                    "description": "Standard authenticated Cloudly access for this hosted instance."
+                  }
+                ],
+                "environmentVariables": {
+                  "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER",
+                  "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID",
+                  "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET",
+                  "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI",
+                  "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE"
+                },
+                "clientAuthenticationMethod": "client_secret_basic"
+              },
+              "minOneboxVersion": "33.8.0",
+              "changelog": "Declare Cloudly's runtime log ledger non-backup data without moving the image: this template runs the Cloudly 33.21.1 image of 1.29.5 and differs from 1.29.5 only in its changelog and two members. databaseBackupExclusions names the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. Every Cloudly start writes the log record of each running attempt that has none before it serves node requests, so the nodes' next batches are kept again after a restore, and the first batch of each capture states the numbers the restore lost as a gap. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.5; a host without the object-storage feature keeps 1.28.4. From 1.29.5 the upgrade moves the template only: Cloudly runs the same image on the same data and no data step runs. From 1.30.0 it moves Cloudly from 33.21.0 to 33.21.1 and keeps the declaration; no data step runs. From 1.29.4 or earlier the template also gains the declaration and the upgrade takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5 and holds for this template: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
+              "breaking": true,
+              "requiresManualReview": true,
+              "migrationRequired": true,
+              "backupBeforeUpgrade": true,
+              "requiresFeatures": [
+                "storage.database.backup-exclusions",
+                "storage.object-storage.backup-exclusions"
+              ],
+              "healthCheck": {
+                "path": "/status",
+                "port": 3000,
+                "expectedStatus": 200
+              }
+            },
+            "configDigest": "sha256:7b7e2f3b78eeacd5319afa6d054cca426e3eb719e309540e83d1fd0176303993"
+          },
+          {
+            "version": "1.30.2",
+            "config": {
+              "image": "code.foss.global/serve.zone/cloudly@sha256:f9859647487308daa9bbd136d02c7fafd651f17265feb7a68cf01212cc7695e1",
               "targetPorts": [
                 {
                   "name": "web",
@@ -10400,7 +11224,7 @@
                 "clientAuthenticationMethod": "client_secret_basic"
               },
               "minOneboxVersion": "33.8.0",
-              "changelog": "Move Cloudly to 33.21.0, a feature release over 33.20.4: Cloudly's runtime log ledger is no longer part of its backups, and a restore keeps the logs of every running workload. This template declares the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, as databaseBackupExclusions of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. The ledger holds most of a long-running Cloudly's database, so its backups shrink to the state they protect. Every Cloudly 33.21.0 start writes the log record of each running attempt that has none, before it serves node requests, so the nodes' next batches are kept again instead of being refused as attempts whose logs Cloudly does not keep; the first batch of each capture after a restore states the numbers the restore lost as a gap. A start whose attempts all have their records writes nothing, and an attempt whose record cannot be written is named in a warning and tried again at the next start. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.4; a host without the object-storage feature keeps 1.28.4. Like 1.29.4, this template declares Cloudly's workload log objects non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3). Cloudly 33.21.0 adds no data step and no index and keeps the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection, and the template requires Onebox 33.8.0. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.0. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
+              "changelog": "Declare Cloudly's runtime log ledger non-backup data without moving the image: this template runs the Cloudly 33.21.2 image of 1.29.6 and differs from 1.29.6 only in its changelog and two members. databaseBackupExclusions names the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. Every Cloudly start writes the log record of each running attempt that has none before it serves node requests, so the nodes' next batches are kept again after a restore, and the first batch of each capture states the numbers the restore lost as a gap. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.6; a host without the object-storage feature keeps 1.28.4. From 1.29.6 the upgrade moves the template only: Cloudly runs the same image on the same data and no data step runs. From 1.30.1 it moves Cloudly from 33.21.1 to 33.21.2 and keeps the declaration; no data step runs. From 1.30.0 it moves Cloudly from 33.21.0 to 33.21.2 and keeps the declaration; no data step runs. From 1.29.5 or earlier the template also gains the declaration and the upgrade takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.6 and holds for this template: Move Cloudly to 33.21.2, a fixes-only patch over 33.21.1, on the config of 1.29.5: 1.29.6 differs from 1.29.5 only in its image and changelog. A recovery redeploy over a dead retained rollout plan, one that no runtime can execute (such as a Swarm-era plan) and whose operation is settled, is no longer refused PLATFORM_CAPABILITY_UNAVAILABLE by its reserve and promote preflights while its Corestore bindings wait for their credentials: such a service has no plan a runtime can run, so it is issued no credential grant until a promotion writes one, and its database and object-storage bindings could not become ready before the promotion those preflights refused. The reserve and promote preflights now defer the readiness of such a binding while it is requested or provisioning, as the promote preflight of a held service's first promotion already did, and warn PLATFORM_BINDING_READINESS_DEFERRED; a binding that adopts a legacy resource is deferred only while Corestore lists that resource as authorized for it, and the route preflight after the rollout still requires every binding ready. The retained plan is judged by the operation it names, as the promotion fence reads it, never by the service's latest operation. 1.29.6 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.6 still exports the ledger with its documents; 1.30.2 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.6 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.2, and 1.29.6 is never the upgrade target of 1.30.0 or 1.30.1, which sort above it. Cloudly 33.21.2 adds no data step, no index and no setting and keeps the data version 0.31.0, so an upgrade from 1.29.5, 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.5 and requires Onebox 33.8.0. Like 1.29.5, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.5 the upgrade moves Cloudly from 33.21.1 to 33.21.2. From 1.29.4 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5, the Cloudly 33.21.1 template, and describes the changes of Cloudly 33.21.1 and the releases before it that an upgrade from 1.29.4 or earlier also takes: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
               "breaking": true,
               "requiresManualReview": true,
               "migrationRequired": true,
@@ -10415,12 +11239,12 @@
                 "expectedStatus": 200
               }
             },
-            "configDigest": "sha256:e97c235d8f27d9210d9828a83a61035a7b42b11cba667826f3c7e3dfb37b813c"
+            "configDigest": "sha256:895f2947850dd0b645c4e1cae8a0bf7a64fe67e491c4396abe6ddd8115634abc"
           },
           {
-            "version": "1.30.1",
+            "version": "1.30.3",
             "config": {
-              "image": "code.foss.global/serve.zone/cloudly@sha256:1e92473c17522967711e8a82459fd8933b8c53b6bad26d0dba516cb9b1eccb34",
+              "image": "code.foss.global/serve.zone/cloudly@sha256:81b959285ed18de92aaba8c6c84c64e1c7ae618af39d3905ef229d3c331807f6",
               "targetPorts": [
                 {
                   "name": "web",
@@ -10604,7 +11428,7 @@
                 "clientAuthenticationMethod": "client_secret_basic"
               },
               "minOneboxVersion": "33.8.0",
-              "changelog": "Declare Cloudly's runtime log ledger non-backup data without moving the image: this template runs the Cloudly 33.21.1 image of 1.29.5 and differs from 1.29.5 only in its changelog and two members. databaseBackupExclusions names the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. Every Cloudly start writes the log record of each running attempt that has none before it serves node requests, so the nodes' next batches are kept again after a restore, and the first batch of each capture states the numbers the restore lost as a gap. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.5; a host without the object-storage feature keeps 1.28.4. From 1.29.5 the upgrade moves the template only: Cloudly runs the same image on the same data and no data step runs. From 1.30.0 it moves Cloudly from 33.21.0 to 33.21.1 and keeps the declaration; no data step runs. From 1.29.4 or earlier the template also gains the declaration and the upgrade takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5 and holds for this template: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
+              "changelog": "Declare Cloudly's runtime log ledger non-backup data without moving the image: this template runs the Cloudly 33.21.3 image of 1.29.7 and differs from 1.29.7 only in its changelog and two members. databaseBackupExclusions names the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. Every Cloudly start writes the log record of each running attempt that has none before it serves node requests, so the nodes' next batches are kept again after a restore, and the first batch of each capture states the numbers the restore lost as a gap. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.7; a host without the object-storage feature keeps 1.28.4. From 1.29.7 the upgrade moves the template only: Cloudly runs the same image on the same data and no data step runs. From 1.30.2 it moves Cloudly from 33.21.2 to 33.21.3, keeps the declaration and runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. From 1.30.1 it moves Cloudly from 33.21.1 to 33.21.3, keeps the declaration and runs that step. From 1.30.0 it moves Cloudly from 33.21.0 to 33.21.3, keeps the declaration and runs that step. Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. From 1.29.6 or earlier the template also gains the declaration and the upgrade takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.7 and holds for this template: Move Cloudly to 33.21.3, a patch over 33.21.2 that adds one forward-only data step, on the config of 1.29.6: 1.29.7 differs from 1.29.6 only in its image and changelog. A service migrated from Swarm reaches its object storage at its node's Corestore endpoint again. The Secrets v2 cutover turned the Docker-era Corestore binding environment, the nine object-storage endpoint names and, for a binding with retention, the eleven S3_RETENTION_* names, into service-owned secrets in scope platform:cloudly-corestore, the endpoint names with the Docker host corestore; a Corestore binding now gives the endpoint names as plain environment composed from the node's endpoint, and the secrets the cutover left behind replaced that environment in the workload, since a secret wins over an inherited variable of the same name, so such a service exited at startup with getaddrinfo ENOTFOUND corestore. Cloudly 33.21.3 adds the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. It retires every active service-owned secret in that scope whose key is one of the twenty non-secret object-storage names (the nine endpoint names and the eleven S3_RETENTION_* names) as a lifecycle change retires one: the version is retired and kept, never purged, and the metadata names the step as the author. The service's secrets revision advances once, and its desired manifests are published again without those secrets under a new secret rollout, the previous one accepted for five more minutes, so the runtime replaces the service's replicas once; a single-replica service restarts briefly. A service whose image plan or secret deployment plan the runtime already refuses gets its manifest from its next promotion. The six credential names, other scopes, other owners and greenfield services are left alone; a database binding gives no plain name, so no database secret is touched. Each service is one transaction, so a rerun writes nothing. The step has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. Cloudly 33.21.3 adds no index, no collection, no setting and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.6 and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and every upgrade to this template runs the step to 0.32.0. 1.29.7 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.7 still exports the ledger with its documents; 1.30.3 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.7 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.3, and 1.29.7 is never the upgrade target of 1.30.0, 1.30.1 or 1.30.2, which sort above it. Like 1.29.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.6 the upgrade moves Cloudly from 33.21.2 to 33.21.3 and runs the step to 0.32.0. From 1.29.5 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.6, the Cloudly 33.21.2 template, and describes the changes of Cloudly 33.21.2 and the releases before it that an upgrade from 1.29.5 or earlier also takes: Move Cloudly to 33.21.2, a fixes-only patch over 33.21.1, on the config of 1.29.5: 1.29.6 differs from 1.29.5 only in its image and changelog. A recovery redeploy over a dead retained rollout plan, one that no runtime can execute (such as a Swarm-era plan) and whose operation is settled, is no longer refused PLATFORM_CAPABILITY_UNAVAILABLE by its reserve and promote preflights while its Corestore bindings wait for their credentials: such a service has no plan a runtime can run, so it is issued no credential grant until a promotion writes one, and its database and object-storage bindings could not become ready before the promotion those preflights refused. The reserve and promote preflights now defer the readiness of such a binding while it is requested or provisioning, as the promote preflight of a held service's first promotion already did, and warn PLATFORM_BINDING_READINESS_DEFERRED; a binding that adopts a legacy resource is deferred only while Corestore lists that resource as authorized for it, and the route preflight after the rollout still requires every binding ready. The retained plan is judged by the operation it names, as the promotion fence reads it, never by the service's latest operation. 1.29.6 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.6 still exports the ledger with its documents; 1.30.2 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.6 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.2, and 1.29.6 is never the upgrade target of 1.30.0 or 1.30.1, which sort above it. Cloudly 33.21.2 adds no data step, no index and no setting and keeps the data version 0.31.0, so an upgrade from 1.29.5, 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.5 and requires Onebox 33.8.0. Like 1.29.5, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.5 the upgrade moves Cloudly from 33.21.1 to 33.21.2. From 1.29.4 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5, the Cloudly 33.21.1 template, and describes the changes of Cloudly 33.21.1 and the releases before it that an upgrade from 1.29.4 or earlier also takes: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
               "breaking": true,
               "requiresManualReview": true,
               "migrationRequired": true,
@@ -10619,12 +11443,12 @@
                 "expectedStatus": 200
               }
             },
-            "configDigest": "sha256:7b7e2f3b78eeacd5319afa6d054cca426e3eb719e309540e83d1fd0176303993"
+            "configDigest": "sha256:ff1a4b84dfc7d931752c5c2ab1a8e6f9165c0f87601c88306f8e045c1c79bdd6"
           },
           {
-            "version": "1.30.2",
+            "version": "1.30.4",
             "config": {
-              "image": "code.foss.global/serve.zone/cloudly@sha256:f9859647487308daa9bbd136d02c7fafd651f17265feb7a68cf01212cc7695e1",
+              "image": "code.foss.global/serve.zone/cloudly@sha256:e8e0f2b5a9e3d7aff4ba3201e358e99900b5c3792c24873b3dc0b3cb3f921439",
               "targetPorts": [
                 {
                   "name": "web",
@@ -10755,6 +11579,12 @@
                   "value": "${S3_SECRET_KEY}",
                   "description": "S3 secret key provisioned by Onebox Corestore.",
                   "required": true
+                },
+                {
+                  "key": "CLOUDLY_BACKUP_CRON",
+                  "value": "15 2 * * *",
+                  "description": "Schedule of Cloudly's backup-all-services task, which backs up every workload service's databases (not its object storage); Cloudly registers the task only while this is set. Daily at 02:15 UTC, clear of Onebox's backup-archive-prune at 03:00.",
+                  "required": false
                 }
               ],
               "platformRequirements": {
@@ -10808,7 +11638,7 @@
                 "clientAuthenticationMethod": "client_secret_basic"
               },
               "minOneboxVersion": "33.8.0",
-              "changelog": "Declare Cloudly's runtime log ledger non-backup data without moving the image: this template runs the Cloudly 33.21.2 image of 1.29.6 and differs from 1.29.6 only in its changelog and two members. databaseBackupExclusions names the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. Every Cloudly start writes the log record of each running attempt that has none before it serves node requests, so the nodes' next batches are kept again after a restore, and the first batch of each capture states the numbers the restore lost as a gap. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.6; a host without the object-storage feature keeps 1.28.4. From 1.29.6 the upgrade moves the template only: Cloudly runs the same image on the same data and no data step runs. From 1.30.1 it moves Cloudly from 33.21.1 to 33.21.2 and keeps the declaration; no data step runs. From 1.30.0 it moves Cloudly from 33.21.0 to 33.21.2 and keeps the declaration; no data step runs. From 1.29.5 or earlier the template also gains the declaration and the upgrade takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.6 and holds for this template: Move Cloudly to 33.21.2, a fixes-only patch over 33.21.1, on the config of 1.29.5: 1.29.6 differs from 1.29.5 only in its image and changelog. A recovery redeploy over a dead retained rollout plan, one that no runtime can execute (such as a Swarm-era plan) and whose operation is settled, is no longer refused PLATFORM_CAPABILITY_UNAVAILABLE by its reserve and promote preflights while its Corestore bindings wait for their credentials: such a service has no plan a runtime can run, so it is issued no credential grant until a promotion writes one, and its database and object-storage bindings could not become ready before the promotion those preflights refused. The reserve and promote preflights now defer the readiness of such a binding while it is requested or provisioning, as the promote preflight of a held service's first promotion already did, and warn PLATFORM_BINDING_READINESS_DEFERRED; a binding that adopts a legacy resource is deferred only while Corestore lists that resource as authorized for it, and the route preflight after the rollout still requires every binding ready. The retained plan is judged by the operation it names, as the promotion fence reads it, never by the service's latest operation. 1.29.6 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.6 still exports the ledger with its documents; 1.30.2 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.6 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.2, and 1.29.6 is never the upgrade target of 1.30.0 or 1.30.1, which sort above it. Cloudly 33.21.2 adds no data step, no index and no setting and keeps the data version 0.31.0, so an upgrade from 1.29.5, 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.5 and requires Onebox 33.8.0. Like 1.29.5, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.5 the upgrade moves Cloudly from 33.21.1 to 33.21.2. From 1.29.4 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5, the Cloudly 33.21.1 template, and describes the changes of Cloudly 33.21.1 and the releases before it that an upgrade from 1.29.4 or earlier also takes: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
+              "changelog": "Declare Cloudly's runtime log ledger non-backup data without moving the image: this template runs the Cloudly 33.21.4 image of 1.29.8 and differs from 1.29.8 only in its changelog and two members. databaseBackupExclusions names the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. Every Cloudly start writes the log record of each running attempt that has none before it serves node requests, so the nodes' next batches are kept again after a restore, and the first batch of each capture states the numbers the restore lost as a gap. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.8; a host without the object-storage feature keeps 1.28.4. Like 1.29.8, this template sets CLOUDLY_BACKUP_CRON to 15 2 * * *, so Cloudly backs up the databases of its workload services daily at 02:15 UTC (not their object storage, AGL-473). From 1.29.8 the upgrade moves the template only: Cloudly runs the same image on the same data and no data step runs. From 1.30.3 it moves Cloudly from 33.21.3 to 33.21.4, keeps the declaration, turns the scheduled workload backups on and runs no data step. From 1.30.2 it moves Cloudly from 33.21.2 to 33.21.4, keeps the declaration, turns the scheduled workload backups on and runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. From 1.30.1 it moves Cloudly from 33.21.1 to 33.21.4, keeps the declaration, turns them on and runs that step. From 1.30.0 it moves Cloudly from 33.21.0 to 33.21.4, keeps the declaration, turns them on and runs that step. Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. From 1.29.7 or earlier the template also gains the declaration and the upgrade takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.8 and holds for this template: Move Cloudly to 33.21.4, a fixes-only patch over 33.21.3, on the config of 1.29.7, and turn on Cloudly's scheduled workload backups: 1.29.8 differs from 1.29.7 only in its image, its changelog and the environment variable CLOUDLY_BACKUP_CRON. Cloudly registers its backup-all-services task only when CLOUDLY_BACKUP_CRON is set, and no earlier template set it, so the databases of the workload services Cloudly runs were never backed up on a schedule. This template sets it to 15 2 * * *: every day at 02:15 UTC, clear of Onebox's backup-archive-prune at 03:00, Cloudly creates a backup of every workload service with backup-enabled resources, and it keeps the newest 24 settled backups of each service (CLOUDLY_BACKUP_KEEP_LAST, which the template does not set, defaults to 24) and deletes older ones with their stored archives. These backups cover the workload databases; a workload's object storage is not part of them (AGL-473). The variable is a plain declaration with no secret and no delivery and is not required. A host that upgrades a service gives a key the service does not have yet the template's value, so an upgraded Cloudly runs the schedule; an operator's stored value for the key wins over the template's. Cloudly 33.21.4 derives the owner of a launcher-environment secret file from the identity the workload runs as (AGL-459): every launcher delivery was published 0:0 and 0400, so a workload that runs as another user could not read its secrets and WorkloadInit exited with SECRET_SOURCE. The manifest publication now derives the owner per target cluster from the spec's runAs and the pinned image's config.User with @serve.zone/interfaces 32.60.0 deriveLauncherSecretDeliveryOwner, an owner that does not derive is refused launcher-secret-ownership-unknown, and the manifests 33.21.3 published are published again on upgrade. On @serve.zone/interfaces 32.60.0, external registry answers no longer carry the stored credential, Test connection proves the credentials through the Docker Registry v2 login, and a service update merges only the fields it states, so Edit and Configure Mail work again for a service with a runtime spec or an immutable image. Cloudly 33.21.4 also fixes the dashboard (AGL-387 to AGL-401: the status bar, the fonts, the dropdown presets, the view actions, the Secrets wizard and index, the Logs view and the App Store view), refuses a deployment without an existing service and node, a node action the node can never receive and a Jump code for a cluster without a relay, and bootstraps a node with the pinned Spark 35.1.18 release. Cloudly 33.21.4 adds no data step and keeps the data version 0.32.0, so an upgrade from 1.29.7 runs no data step; it adds no index, no collection and no setting, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is that of 1.29.7 with CLOUDLY_BACKUP_CRON and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and an upgrade from 1.29.6 or earlier runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0, which has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. 1.29.8 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.8 still exports the ledger with its documents; 1.30.4 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.8 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.4, and 1.29.8 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2 or 1.30.3, which sort above it. Like 1.29.7, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.7 the upgrade moves Cloudly from 33.21.3 to 33.21.4 and turns the scheduled workload backups on. From 1.29.6 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.7, the Cloudly 33.21.3 template, and describes the changes of Cloudly 33.21.3 and the releases before it that an upgrade from 1.29.6 or earlier also takes: Move Cloudly to 33.21.3, a patch over 33.21.2 that adds one forward-only data step, on the config of 1.29.6: 1.29.7 differs from 1.29.6 only in its image and changelog. A service migrated from Swarm reaches its object storage at its node's Corestore endpoint again. The Secrets v2 cutover turned the Docker-era Corestore binding environment, the nine object-storage endpoint names and, for a binding with retention, the eleven S3_RETENTION_* names, into service-owned secrets in scope platform:cloudly-corestore, the endpoint names with the Docker host corestore; a Corestore binding now gives the endpoint names as plain environment composed from the node's endpoint, and the secrets the cutover left behind replaced that environment in the workload, since a secret wins over an inherited variable of the same name, so such a service exited at startup with getaddrinfo ENOTFOUND corestore. Cloudly 33.21.3 adds the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. It retires every active service-owned secret in that scope whose key is one of the twenty non-secret object-storage names (the nine endpoint names and the eleven S3_RETENTION_* names) as a lifecycle change retires one: the version is retired and kept, never purged, and the metadata names the step as the author. The service's secrets revision advances once, and its desired manifests are published again without those secrets under a new secret rollout, the previous one accepted for five more minutes, so the runtime replaces the service's replicas once; a single-replica service restarts briefly. A service whose image plan or secret deployment plan the runtime already refuses gets its manifest from its next promotion. The six credential names, other scopes, other owners and greenfield services are left alone; a database binding gives no plain name, so no database secret is touched. Each service is one transaction, so a rerun writes nothing. The step has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. Cloudly 33.21.3 adds no index, no collection, no setting and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.6 and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and every upgrade to this template runs the step to 0.32.0. 1.29.7 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.7 still exports the ledger with its documents; 1.30.3 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.7 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.3, and 1.29.7 is never the upgrade target of 1.30.0, 1.30.1 or 1.30.2, which sort above it. Like 1.29.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.6 the upgrade moves Cloudly from 33.21.2 to 33.21.3 and runs the step to 0.32.0. From 1.29.5 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.6, the Cloudly 33.21.2 template, and describes the changes of Cloudly 33.21.2 and the releases before it that an upgrade from 1.29.5 or earlier also takes: Move Cloudly to 33.21.2, a fixes-only patch over 33.21.1, on the config of 1.29.5: 1.29.6 differs from 1.29.5 only in its image and changelog. A recovery redeploy over a dead retained rollout plan, one that no runtime can execute (such as a Swarm-era plan) and whose operation is settled, is no longer refused PLATFORM_CAPABILITY_UNAVAILABLE by its reserve and promote preflights while its Corestore bindings wait for their credentials: such a service has no plan a runtime can run, so it is issued no credential grant until a promotion writes one, and its database and object-storage bindings could not become ready before the promotion those preflights refused. The reserve and promote preflights now defer the readiness of such a binding while it is requested or provisioning, as the promote preflight of a held service's first promotion already did, and warn PLATFORM_BINDING_READINESS_DEFERRED; a binding that adopts a legacy resource is deferred only while Corestore lists that resource as authorized for it, and the route preflight after the rollout still requires every binding ready. The retained plan is judged by the operation it names, as the promotion fence reads it, never by the service's latest operation. 1.29.6 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.6 still exports the ledger with its documents; 1.30.2 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.6 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.2, and 1.29.6 is never the upgrade target of 1.30.0 or 1.30.1, which sort above it. Cloudly 33.21.2 adds no data step, no index and no setting and keeps the data version 0.31.0, so an upgrade from 1.29.5, 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.5 and requires Onebox 33.8.0. Like 1.29.5, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.5 the upgrade moves Cloudly from 33.21.1 to 33.21.2. From 1.29.4 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5, the Cloudly 33.21.1 template, and describes the changes of Cloudly 33.21.1 and the releases before it that an upgrade from 1.29.4 or earlier also takes: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
               "breaking": true,
               "requiresManualReview": true,
               "migrationRequired": true,
@@ -10823,12 +11653,12 @@
                 "expectedStatus": 200
               }
             },
-            "configDigest": "sha256:895f2947850dd0b645c4e1cae8a0bf7a64fe67e491c4396abe6ddd8115634abc"
+            "configDigest": "sha256:61778445373bd108eac47584385e2a82558cb742942f0d96d065554c8ad1ac4b"
           },
           {
-            "version": "1.30.3",
+            "version": "1.30.5",
             "config": {
-              "image": "code.foss.global/serve.zone/cloudly@sha256:81b959285ed18de92aaba8c6c84c64e1c7ae618af39d3905ef229d3c331807f6",
+              "image": "code.foss.global/serve.zone/cloudly@sha256:73be2fb57ee6bf40f83569c677f381713a7ddcc38fedbb3f16a7d85fb1e6e33d",
               "targetPorts": [
                 {
                   "name": "web",
@@ -10959,6 +11789,12 @@
                   "value": "${S3_SECRET_KEY}",
                   "description": "S3 secret key provisioned by Onebox Corestore.",
                   "required": true
+                },
+                {
+                  "key": "CLOUDLY_BACKUP_CRON",
+                  "value": "15 2 * * *",
+                  "description": "Schedule of Cloudly's backup-all-services task, which backs up every workload service's databases (not its object storage); Cloudly registers the task only while this is set. Daily at 02:15 UTC, clear of Onebox's backup-archive-prune at 03:00.",
+                  "required": false
                 }
               ],
               "platformRequirements": {
@@ -11012,7 +11848,7 @@
                 "clientAuthenticationMethod": "client_secret_basic"
               },
               "minOneboxVersion": "33.8.0",
-              "changelog": "Declare Cloudly's runtime log ledger non-backup data without moving the image: this template runs the Cloudly 33.21.3 image of 1.29.7 and differs from 1.29.7 only in its changelog and two members. databaseBackupExclusions names the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. Every Cloudly start writes the log record of each running attempt that has none before it serves node requests, so the nodes' next batches are kept again after a restore, and the first batch of each capture states the numbers the restore lost as a gap. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.7; a host without the object-storage feature keeps 1.28.4. From 1.29.7 the upgrade moves the template only: Cloudly runs the same image on the same data and no data step runs. From 1.30.2 it moves Cloudly from 33.21.2 to 33.21.3, keeps the declaration and runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. From 1.30.1 it moves Cloudly from 33.21.1 to 33.21.3, keeps the declaration and runs that step. From 1.30.0 it moves Cloudly from 33.21.0 to 33.21.3, keeps the declaration and runs that step. Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. From 1.29.6 or earlier the template also gains the declaration and the upgrade takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.7 and holds for this template: Move Cloudly to 33.21.3, a patch over 33.21.2 that adds one forward-only data step, on the config of 1.29.6: 1.29.7 differs from 1.29.6 only in its image and changelog. A service migrated from Swarm reaches its object storage at its node's Corestore endpoint again. The Secrets v2 cutover turned the Docker-era Corestore binding environment, the nine object-storage endpoint names and, for a binding with retention, the eleven S3_RETENTION_* names, into service-owned secrets in scope platform:cloudly-corestore, the endpoint names with the Docker host corestore; a Corestore binding now gives the endpoint names as plain environment composed from the node's endpoint, and the secrets the cutover left behind replaced that environment in the workload, since a secret wins over an inherited variable of the same name, so such a service exited at startup with getaddrinfo ENOTFOUND corestore. Cloudly 33.21.3 adds the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. It retires every active service-owned secret in that scope whose key is one of the twenty non-secret object-storage names (the nine endpoint names and the eleven S3_RETENTION_* names) as a lifecycle change retires one: the version is retired and kept, never purged, and the metadata names the step as the author. The service's secrets revision advances once, and its desired manifests are published again without those secrets under a new secret rollout, the previous one accepted for five more minutes, so the runtime replaces the service's replicas once; a single-replica service restarts briefly. A service whose image plan or secret deployment plan the runtime already refuses gets its manifest from its next promotion. The six credential names, other scopes, other owners and greenfield services are left alone; a database binding gives no plain name, so no database secret is touched. Each service is one transaction, so a rerun writes nothing. The step has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. Cloudly 33.21.3 adds no index, no collection, no setting and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.6 and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and every upgrade to this template runs the step to 0.32.0. 1.29.7 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.7 still exports the ledger with its documents; 1.30.3 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.7 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.3, and 1.29.7 is never the upgrade target of 1.30.0, 1.30.1 or 1.30.2, which sort above it. Like 1.29.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.6 the upgrade moves Cloudly from 33.21.2 to 33.21.3 and runs the step to 0.32.0. From 1.29.5 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.6, the Cloudly 33.21.2 template, and describes the changes of Cloudly 33.21.2 and the releases before it that an upgrade from 1.29.5 or earlier also takes: Move Cloudly to 33.21.2, a fixes-only patch over 33.21.1, on the config of 1.29.5: 1.29.6 differs from 1.29.5 only in its image and changelog. A recovery redeploy over a dead retained rollout plan, one that no runtime can execute (such as a Swarm-era plan) and whose operation is settled, is no longer refused PLATFORM_CAPABILITY_UNAVAILABLE by its reserve and promote preflights while its Corestore bindings wait for their credentials: such a service has no plan a runtime can run, so it is issued no credential grant until a promotion writes one, and its database and object-storage bindings could not become ready before the promotion those preflights refused. The reserve and promote preflights now defer the readiness of such a binding while it is requested or provisioning, as the promote preflight of a held service's first promotion already did, and warn PLATFORM_BINDING_READINESS_DEFERRED; a binding that adopts a legacy resource is deferred only while Corestore lists that resource as authorized for it, and the route preflight after the rollout still requires every binding ready. The retained plan is judged by the operation it names, as the promotion fence reads it, never by the service's latest operation. 1.29.6 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.6 still exports the ledger with its documents; 1.30.2 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.6 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.2, and 1.29.6 is never the upgrade target of 1.30.0 or 1.30.1, which sort above it. Cloudly 33.21.2 adds no data step, no index and no setting and keeps the data version 0.31.0, so an upgrade from 1.29.5, 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.5 and requires Onebox 33.8.0. Like 1.29.5, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.5 the upgrade moves Cloudly from 33.21.1 to 33.21.2. From 1.29.4 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5, the Cloudly 33.21.1 template, and describes the changes of Cloudly 33.21.1 and the releases before it that an upgrade from 1.29.4 or earlier also takes: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
+              "changelog": "Declare Cloudly's runtime log ledger non-backup data without moving the image: this template runs the Cloudly 33.23.0 image of 1.29.9 and differs from 1.29.9 only in its changelog and two members. databaseBackupExclusions names the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. Every Cloudly start writes the log record of each running attempt that has none before it serves node requests, so the nodes' next batches are kept again after a restore, and the first batch of each capture states the numbers the restore lost as a gap. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.9; a host without the object-storage feature keeps 1.28.4. Like 1.29.9, this template sets CLOUDLY_BACKUP_CRON to 15 2 * * *, so Cloudly backs up the databases of its workload services daily at 02:15 UTC (not their object storage, AGL-473). From 1.29.9 the upgrade moves the template only: Cloudly runs the same image on the same data and no data step runs. From 1.30.4 it moves Cloudly from 33.21.4 to 33.23.0, keeps the declaration and runs the forward-only data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0. From 1.30.3 it moves Cloudly from 33.21.3 to 33.23.0, keeps the declaration, turns the scheduled workload backups on and runs that step. From 1.30.2 it moves Cloudly from 33.21.2 to 33.23.0, keeps the declaration, turns them on and runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0 and then that step. From 1.30.1 it moves Cloudly from 33.21.1 to 33.23.0 and from 1.30.0 from 33.21.0, keeps the declaration, turns them on and runs both steps. Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. From 1.29.8 or earlier the template also gains the declaration and the upgrade takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.9 and holds for this template: Move Cloudly to 33.23.0 on the config of 1.29.8: 1.29.9 differs from 1.29.8 only in its image and changelog. Cloudly 33.23.0 adds one forward-only data step, retire-stale-reconciliation-stages from 0.32.0 to 0.33.0, and takes the changes of Cloudly 33.22.0, which no template ran. A service's status is now what the producers Cloudly runs state about it: the Pallet runtime's verdict on each target, the current plan's rollout, the newest deployment operation and the records of the observers Cloudly runs (externalGateway, externalGatewayMail, coreMail and platformBinding). The Services list, the Overview and the MCP tool cloudly_list_services read that one status, so a service that showed Failed or Converging for good because of a stage the Swarm-era Coreflow reconciler last reported on 2026-08-03 shows its real state. getServices composes the status for an administrator only; a cluster's relay gets the services without it. The Overview lists every failed service with its reason and a link to it (degraded ones as warnings); its Deployments tile and Recent deployment changes table are gone, and so is the dashboard summary's deployments count. Cloudly no longer serves reportReconciliationStatus and reportDeploymentSnapshots, the Swarm-era Coreflow intake no component sends, and an observer deletes its record when what it observes goes: a route withdrawn from a service and a deleted or tombstoned platform binding withdraw their externalGateway and platformBinding record. The data step deletes, through the database driver before any model is prepared, every reconciliation record of a kind only the Swarm-era Coreflow reconciler wrote (workload, routing, immutable-image, immutable-adoption and immutable-attestation) and every externalGateway record of a route its service no longer declares; a record of a route still declared stays, failed or not, so do the records of other observers, and a rerun removes nothing. The deployment operations get the index service_deployment_operation_newest, which the model builds at start. Cloudly 33.23.0 adds delegated admin temp tokens: a human administrator who signed in within five minutes mints a token for the scopes secrets and serviceCredentials that lives one hour at most (servezone admin temp-token mint, list and revoke); Cloudly stores only its SHA-256, in cloudly_delegated_admin_tokens, records every use, refusal and failed write in cloudly_delegated_admin_token_audit, and a token never writes a reserved name and never deletes. Cloudly 33.23.0 moves to @serve.zone/interfaces 32.61.0 and its dashboard to @design.estate/dees-catalog 23 (controls are 30px tall instead of 32px, dialogs and sheets have 14px corners instead of 20px). With Cloudly 33.22.0 the scheduled backup-all-services task, which 1.29.8 turned on with CLOUDLY_BACKUP_CRON, backs up each workload service on the node and cluster its data lives on: on a Cloudly with more than one cluster every service was refused, and on one with a single cluster every backup was refused backup-node-required, while the run was recorded completed. A service with nothing to back up is skipped, a service without a target fails by name, and a run in which any service failed ends failed and names each one. Cloudly 33.22.0 adds servezone service credential inspect, ensure, rotate and revoke for the service machine credential @serve.zone/platformclient 4.0.0 and later signs in with, and names a refused actor SERVICE_CREDENTIAL_UNAUTHORIZED. Neither release adds an environment variable or a setting, and neither changes the Dockerfile or the qenv.yml requirements, so 1.29.9 keeps 1.29.8's runtime contract, CLOUDLY_BACKUP_CRON 15 2 * * * included, and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: the data step has no way back. Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.29.8, 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. 1.29.9 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.9 still exports the ledger with its documents; 1.30.5 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.9 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.5, and 1.29.9 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2, 1.30.3 or 1.30.4, which sort above it. Like 1.29.8, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.8 the upgrade moves Cloudly from 33.21.4 to 33.23.0 and runs the data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0. From 1.29.7 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.8, the Cloudly 33.21.4 template, and describes the changes of Cloudly 33.21.4 and the releases before it that an upgrade from 1.29.7 or earlier also takes: Move Cloudly to 33.21.4, a fixes-only patch over 33.21.3, on the config of 1.29.7, and turn on Cloudly's scheduled workload backups: 1.29.8 differs from 1.29.7 only in its image, its changelog and the environment variable CLOUDLY_BACKUP_CRON. Cloudly registers its backup-all-services task only when CLOUDLY_BACKUP_CRON is set, and no earlier template set it, so the databases of the workload services Cloudly runs were never backed up on a schedule. This template sets it to 15 2 * * *: every day at 02:15 UTC, clear of Onebox's backup-archive-prune at 03:00, Cloudly creates a backup of every workload service with backup-enabled resources, and it keeps the newest 24 settled backups of each service (CLOUDLY_BACKUP_KEEP_LAST, which the template does not set, defaults to 24) and deletes older ones with their stored archives. These backups cover the workload databases; a workload's object storage is not part of them (AGL-473). The variable is a plain declaration with no secret and no delivery and is not required. A host that upgrades a service gives a key the service does not have yet the template's value, so an upgraded Cloudly runs the schedule; an operator's stored value for the key wins over the template's. Cloudly 33.21.4 derives the owner of a launcher-environment secret file from the identity the workload runs as (AGL-459): every launcher delivery was published 0:0 and 0400, so a workload that runs as another user could not read its secrets and WorkloadInit exited with SECRET_SOURCE. The manifest publication now derives the owner per target cluster from the spec's runAs and the pinned image's config.User with @serve.zone/interfaces 32.60.0 deriveLauncherSecretDeliveryOwner, an owner that does not derive is refused launcher-secret-ownership-unknown, and the manifests 33.21.3 published are published again on upgrade. On @serve.zone/interfaces 32.60.0, external registry answers no longer carry the stored credential, Test connection proves the credentials through the Docker Registry v2 login, and a service update merges only the fields it states, so Edit and Configure Mail work again for a service with a runtime spec or an immutable image. Cloudly 33.21.4 also fixes the dashboard (AGL-387 to AGL-401: the status bar, the fonts, the dropdown presets, the view actions, the Secrets wizard and index, the Logs view and the App Store view), refuses a deployment without an existing service and node, a node action the node can never receive and a Jump code for a cluster without a relay, and bootstraps a node with the pinned Spark 35.1.18 release. Cloudly 33.21.4 adds no data step and keeps the data version 0.32.0, so an upgrade from 1.29.7 runs no data step; it adds no index, no collection and no setting, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is that of 1.29.7 with CLOUDLY_BACKUP_CRON and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and an upgrade from 1.29.6 or earlier runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0, which has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. 1.29.8 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.8 still exports the ledger with its documents; 1.30.4 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.8 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.4, and 1.29.8 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2 or 1.30.3, which sort above it. Like 1.29.7, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.7 the upgrade moves Cloudly from 33.21.3 to 33.21.4 and turns the scheduled workload backups on. From 1.29.6 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.7, the Cloudly 33.21.3 template, and describes the changes of Cloudly 33.21.3 and the releases before it that an upgrade from 1.29.6 or earlier also takes: Move Cloudly to 33.21.3, a patch over 33.21.2 that adds one forward-only data step, on the config of 1.29.6: 1.29.7 differs from 1.29.6 only in its image and changelog. A service migrated from Swarm reaches its object storage at its node's Corestore endpoint again. The Secrets v2 cutover turned the Docker-era Corestore binding environment, the nine object-storage endpoint names and, for a binding with retention, the eleven S3_RETENTION_* names, into service-owned secrets in scope platform:cloudly-corestore, the endpoint names with the Docker host corestore; a Corestore binding now gives the endpoint names as plain environment composed from the node's endpoint, and the secrets the cutover left behind replaced that environment in the workload, since a secret wins over an inherited variable of the same name, so such a service exited at startup with getaddrinfo ENOTFOUND corestore. Cloudly 33.21.3 adds the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. It retires every active service-owned secret in that scope whose key is one of the twenty non-secret object-storage names (the nine endpoint names and the eleven S3_RETENTION_* names) as a lifecycle change retires one: the version is retired and kept, never purged, and the metadata names the step as the author. The service's secrets revision advances once, and its desired manifests are published again without those secrets under a new secret rollout, the previous one accepted for five more minutes, so the runtime replaces the service's replicas once; a single-replica service restarts briefly. A service whose image plan or secret deployment plan the runtime already refuses gets its manifest from its next promotion. The six credential names, other scopes, other owners and greenfield services are left alone; a database binding gives no plain name, so no database secret is touched. Each service is one transaction, so a rerun writes nothing. The step has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. Cloudly 33.21.3 adds no index, no collection, no setting and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.6 and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and every upgrade to this template runs the step to 0.32.0. 1.29.7 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.7 still exports the ledger with its documents; 1.30.3 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.7 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.3, and 1.29.7 is never the upgrade target of 1.30.0, 1.30.1 or 1.30.2, which sort above it. Like 1.29.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.6 the upgrade moves Cloudly from 33.21.2 to 33.21.3 and runs the step to 0.32.0. From 1.29.5 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.6, the Cloudly 33.21.2 template, and describes the changes of Cloudly 33.21.2 and the releases before it that an upgrade from 1.29.5 or earlier also takes: Move Cloudly to 33.21.2, a fixes-only patch over 33.21.1, on the config of 1.29.5: 1.29.6 differs from 1.29.5 only in its image and changelog. A recovery redeploy over a dead retained rollout plan, one that no runtime can execute (such as a Swarm-era plan) and whose operation is settled, is no longer refused PLATFORM_CAPABILITY_UNAVAILABLE by its reserve and promote preflights while its Corestore bindings wait for their credentials: such a service has no plan a runtime can run, so it is issued no credential grant until a promotion writes one, and its database and object-storage bindings could not become ready before the promotion those preflights refused. The reserve and promote preflights now defer the readiness of such a binding while it is requested or provisioning, as the promote preflight of a held service's first promotion already did, and warn PLATFORM_BINDING_READINESS_DEFERRED; a binding that adopts a legacy resource is deferred only while Corestore lists that resource as authorized for it, and the route preflight after the rollout still requires every binding ready. The retained plan is judged by the operation it names, as the promotion fence reads it, never by the service's latest operation. 1.29.6 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.6 still exports the ledger with its documents; 1.30.2 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.6 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.2, and 1.29.6 is never the upgrade target of 1.30.0 or 1.30.1, which sort above it. Cloudly 33.21.2 adds no data step, no index and no setting and keeps the data version 0.31.0, so an upgrade from 1.29.5, 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.5 and requires Onebox 33.8.0. Like 1.29.5, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.5 the upgrade moves Cloudly from 33.21.1 to 33.21.2. From 1.29.4 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5, the Cloudly 33.21.1 template, and describes the changes of Cloudly 33.21.1 and the releases before it that an upgrade from 1.29.4 or earlier also takes: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
               "breaking": true,
               "requiresManualReview": true,
               "migrationRequired": true,
@@ -11027,12 +11863,12 @@
                 "expectedStatus": 200
               }
             },
-            "configDigest": "sha256:ff1a4b84dfc7d931752c5c2ab1a8e6f9165c0f87601c88306f8e045c1c79bdd6"
+            "configDigest": "sha256:e30a131ead0cda7dbb4aa799a78ca991d75cf917d9d25abe7c02e604743af19a"
           },
           {
-            "version": "1.30.4",
+            "version": "1.30.6",
             "config": {
-              "image": "code.foss.global/serve.zone/cloudly@sha256:e8e0f2b5a9e3d7aff4ba3201e358e99900b5c3792c24873b3dc0b3cb3f921439",
+              "image": "code.foss.global/serve.zone/cloudly@sha256:cd0808e1fa4fdcf49742ac527480013ca506eb653fea4f82a86de546d1186876",
               "targetPorts": [
                 {
                   "name": "web",
@@ -11222,7 +12058,7 @@
                 "clientAuthenticationMethod": "client_secret_basic"
               },
               "minOneboxVersion": "33.8.0",
-              "changelog": "Declare Cloudly's runtime log ledger non-backup data without moving the image: this template runs the Cloudly 33.21.4 image of 1.29.8 and differs from 1.29.8 only in its changelog and two members. databaseBackupExclusions names the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. Every Cloudly start writes the log record of each running attempt that has none before it serves node requests, so the nodes' next batches are kept again after a restore, and the first batch of each capture states the numbers the restore lost as a gap. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.8; a host without the object-storage feature keeps 1.28.4. Like 1.29.8, this template sets CLOUDLY_BACKUP_CRON to 15 2 * * *, so Cloudly backs up the databases of its workload services daily at 02:15 UTC (not their object storage, AGL-473). From 1.29.8 the upgrade moves the template only: Cloudly runs the same image on the same data and no data step runs. From 1.30.3 it moves Cloudly from 33.21.3 to 33.21.4, keeps the declaration, turns the scheduled workload backups on and runs no data step. From 1.30.2 it moves Cloudly from 33.21.2 to 33.21.4, keeps the declaration, turns the scheduled workload backups on and runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. From 1.30.1 it moves Cloudly from 33.21.1 to 33.21.4, keeps the declaration, turns them on and runs that step. From 1.30.0 it moves Cloudly from 33.21.0 to 33.21.4, keeps the declaration, turns them on and runs that step. Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. From 1.29.7 or earlier the template also gains the declaration and the upgrade takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.8 and holds for this template: Move Cloudly to 33.21.4, a fixes-only patch over 33.21.3, on the config of 1.29.7, and turn on Cloudly's scheduled workload backups: 1.29.8 differs from 1.29.7 only in its image, its changelog and the environment variable CLOUDLY_BACKUP_CRON. Cloudly registers its backup-all-services task only when CLOUDLY_BACKUP_CRON is set, and no earlier template set it, so the databases of the workload services Cloudly runs were never backed up on a schedule. This template sets it to 15 2 * * *: every day at 02:15 UTC, clear of Onebox's backup-archive-prune at 03:00, Cloudly creates a backup of every workload service with backup-enabled resources, and it keeps the newest 24 settled backups of each service (CLOUDLY_BACKUP_KEEP_LAST, which the template does not set, defaults to 24) and deletes older ones with their stored archives. These backups cover the workload databases; a workload's object storage is not part of them (AGL-473). The variable is a plain declaration with no secret and no delivery and is not required. A host that upgrades a service gives a key the service does not have yet the template's value, so an upgraded Cloudly runs the schedule; an operator's stored value for the key wins over the template's. Cloudly 33.21.4 derives the owner of a launcher-environment secret file from the identity the workload runs as (AGL-459): every launcher delivery was published 0:0 and 0400, so a workload that runs as another user could not read its secrets and WorkloadInit exited with SECRET_SOURCE. The manifest publication now derives the owner per target cluster from the spec's runAs and the pinned image's config.User with @serve.zone/interfaces 32.60.0 deriveLauncherSecretDeliveryOwner, an owner that does not derive is refused launcher-secret-ownership-unknown, and the manifests 33.21.3 published are published again on upgrade. On @serve.zone/interfaces 32.60.0, external registry answers no longer carry the stored credential, Test connection proves the credentials through the Docker Registry v2 login, and a service update merges only the fields it states, so Edit and Configure Mail work again for a service with a runtime spec or an immutable image. Cloudly 33.21.4 also fixes the dashboard (AGL-387 to AGL-401: the status bar, the fonts, the dropdown presets, the view actions, the Secrets wizard and index, the Logs view and the App Store view), refuses a deployment without an existing service and node, a node action the node can never receive and a Jump code for a cluster without a relay, and bootstraps a node with the pinned Spark 35.1.18 release. Cloudly 33.21.4 adds no data step and keeps the data version 0.32.0, so an upgrade from 1.29.7 runs no data step; it adds no index, no collection and no setting, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is that of 1.29.7 with CLOUDLY_BACKUP_CRON and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and an upgrade from 1.29.6 or earlier runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0, which has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. 1.29.8 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.8 still exports the ledger with its documents; 1.30.4 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.8 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.4, and 1.29.8 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2 or 1.30.3, which sort above it. Like 1.29.7, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.7 the upgrade moves Cloudly from 33.21.3 to 33.21.4 and turns the scheduled workload backups on. From 1.29.6 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.7, the Cloudly 33.21.3 template, and describes the changes of Cloudly 33.21.3 and the releases before it that an upgrade from 1.29.6 or earlier also takes: Move Cloudly to 33.21.3, a patch over 33.21.2 that adds one forward-only data step, on the config of 1.29.6: 1.29.7 differs from 1.29.6 only in its image and changelog. A service migrated from Swarm reaches its object storage at its node's Corestore endpoint again. The Secrets v2 cutover turned the Docker-era Corestore binding environment, the nine object-storage endpoint names and, for a binding with retention, the eleven S3_RETENTION_* names, into service-owned secrets in scope platform:cloudly-corestore, the endpoint names with the Docker host corestore; a Corestore binding now gives the endpoint names as plain environment composed from the node's endpoint, and the secrets the cutover left behind replaced that environment in the workload, since a secret wins over an inherited variable of the same name, so such a service exited at startup with getaddrinfo ENOTFOUND corestore. Cloudly 33.21.3 adds the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. It retires every active service-owned secret in that scope whose key is one of the twenty non-secret object-storage names (the nine endpoint names and the eleven S3_RETENTION_* names) as a lifecycle change retires one: the version is retired and kept, never purged, and the metadata names the step as the author. The service's secrets revision advances once, and its desired manifests are published again without those secrets under a new secret rollout, the previous one accepted for five more minutes, so the runtime replaces the service's replicas once; a single-replica service restarts briefly. A service whose image plan or secret deployment plan the runtime already refuses gets its manifest from its next promotion. The six credential names, other scopes, other owners and greenfield services are left alone; a database binding gives no plain name, so no database secret is touched. Each service is one transaction, so a rerun writes nothing. The step has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. Cloudly 33.21.3 adds no index, no collection, no setting and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.6 and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and every upgrade to this template runs the step to 0.32.0. 1.29.7 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.7 still exports the ledger with its documents; 1.30.3 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.7 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.3, and 1.29.7 is never the upgrade target of 1.30.0, 1.30.1 or 1.30.2, which sort above it. Like 1.29.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.6 the upgrade moves Cloudly from 33.21.2 to 33.21.3 and runs the step to 0.32.0. From 1.29.5 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.6, the Cloudly 33.21.2 template, and describes the changes of Cloudly 33.21.2 and the releases before it that an upgrade from 1.29.5 or earlier also takes: Move Cloudly to 33.21.2, a fixes-only patch over 33.21.1, on the config of 1.29.5: 1.29.6 differs from 1.29.5 only in its image and changelog. A recovery redeploy over a dead retained rollout plan, one that no runtime can execute (such as a Swarm-era plan) and whose operation is settled, is no longer refused PLATFORM_CAPABILITY_UNAVAILABLE by its reserve and promote preflights while its Corestore bindings wait for their credentials: such a service has no plan a runtime can run, so it is issued no credential grant until a promotion writes one, and its database and object-storage bindings could not become ready before the promotion those preflights refused. The reserve and promote preflights now defer the readiness of such a binding while it is requested or provisioning, as the promote preflight of a held service's first promotion already did, and warn PLATFORM_BINDING_READINESS_DEFERRED; a binding that adopts a legacy resource is deferred only while Corestore lists that resource as authorized for it, and the route preflight after the rollout still requires every binding ready. The retained plan is judged by the operation it names, as the promotion fence reads it, never by the service's latest operation. 1.29.6 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.6 still exports the ledger with its documents; 1.30.2 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.6 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.2, and 1.29.6 is never the upgrade target of 1.30.0 or 1.30.1, which sort above it. Cloudly 33.21.2 adds no data step, no index and no setting and keeps the data version 0.31.0, so an upgrade from 1.29.5, 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.5 and requires Onebox 33.8.0. Like 1.29.5, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.5 the upgrade moves Cloudly from 33.21.1 to 33.21.2. From 1.29.4 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5, the Cloudly 33.21.1 template, and describes the changes of Cloudly 33.21.1 and the releases before it that an upgrade from 1.29.4 or earlier also takes: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
+              "changelog": "Declare Cloudly's runtime log ledger non-backup data without moving the image: this template runs the Cloudly 33.23.2 image of 1.29.10 and differs from 1.29.10 only in its changelog and two members. databaseBackupExclusions names the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. Every Cloudly start writes the log record of each running attempt that has none before it serves node requests, so the nodes' next batches are kept again after a restore, and the first batch of each capture states the numbers the restore lost as a gap. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.10; a host without the object-storage feature keeps 1.28.4. Like 1.29.10, this template sets CLOUDLY_BACKUP_CRON to 15 2 * * *, so Cloudly backs up the databases of its workload services daily at 02:15 UTC (not their object storage, AGL-473). From 1.29.10 the upgrade moves the template only: Cloudly runs the same image on the same data and no data step runs. From 1.30.5 it moves Cloudly from 33.23.0 to 33.23.2, keeps the declaration and runs no data step. From 1.30.4 it moves Cloudly from 33.21.4 to 33.23.2, keeps the declaration and runs the forward-only data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0. From 1.30.3 it moves Cloudly from 33.21.3 to 33.23.2, keeps the declaration, turns the scheduled workload backups on and runs that step. From 1.30.2 it moves Cloudly from 33.21.2 to 33.23.2, keeps the declaration, turns them on and runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0 and then that step. From 1.30.1 it moves Cloudly from 33.21.1 to 33.23.2 and from 1.30.0 from 33.21.0, keeps the declaration, turns them on and runs both steps. Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. From 1.29.9 or earlier the template also gains the declaration and the upgrade takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.10 and holds for this template: Move Cloudly to 33.23.2, a fixes-only patch over 33.23.0, on the config of 1.29.9: 1.29.10 differs from 1.29.9 only in its image and changelog. Cloudly 33.23.2 takes the changes of Cloudly 33.23.1, which no template ran. Cloudly 33.23.1 composes each service's status on its own: a service whose stored image deployment plan, rollout status or newest deployment operation violates the @serve.zone/interfaces contract in a field the status reads gets the status failed with one composition stage that names the violation, for example: Cloudly cannot compose this service's status: its stored image deployment plan violates the contract (releaseTag, operationId missing). A composition that throws gives that service the same failed status with the reason and one error log line naming the service, and every other service keeps its status. Cloudly 33.23.0 answered the administrator's services list, getServiceById of such a service and the MCP tool cloudly_list_services with an internal error when one service stored a plan written before the plan carried releaseTag and operationId. The stored documents are not changed; a new promotion replaces such a plan. Cloudly 33.23.2 lets servezone service credential ensure give a service whose SERVEZONE_PLATFORM_AUTHORIZATION an operator stored as an ordinary secret its service machine credential: ensure, run by a human administrator who signed in within five minutes or with a serviceCredentials temp token, takes that operator-managed row over in the same transaction, active, retired, revoked or deleted, revokes every legacy version that could still be issued, keeps every version, moves the row to the credential's platform scope stamped with the administrator and the token, records the takeover on the new version, publishes the service's secret manifests again and is fenced on the operator authority and revision it read. Before, ensure was refused SERVICE_CREDENTIAL_CONFLICT for such a service whatever an administrator did first. A secret of the key under any other authority is still refused, and the legacy user token itself is not revoked automatically. Neither release adds a data step, an environment variable, a setting, a collection or an index, and neither changes the Dockerfile or the qenv.yml requirements, so 1.29.10 keeps 1.29.9's runtime contract, CLOUDLY_BACKUP_CRON 15 2 * * * included, the data version 0.33.0 and the Onebox minimum 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and an upgrade from 1.29.8 or earlier runs the forward-only data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0, which has no way back: Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.29.8, 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. 1.29.10 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.10 still exports the ledger with its documents; 1.30.6 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.10 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.6, and 1.29.10 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2, 1.30.3, 1.30.4 or 1.30.5, which sort above it. Like 1.29.9, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.9 the upgrade moves Cloudly from 33.23.0 to 33.23.2 and runs no data step. From 1.29.8 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.9, the Cloudly 33.23.0 template, and describes the changes of Cloudly 33.23.0 and the releases before it that an upgrade from 1.29.8 or earlier also takes: Move Cloudly to 33.23.0 on the config of 1.29.8: 1.29.9 differs from 1.29.8 only in its image and changelog. Cloudly 33.23.0 adds one forward-only data step, retire-stale-reconciliation-stages from 0.32.0 to 0.33.0, and takes the changes of Cloudly 33.22.0, which no template ran. A service's status is now what the producers Cloudly runs state about it: the Pallet runtime's verdict on each target, the current plan's rollout, the newest deployment operation and the records of the observers Cloudly runs (externalGateway, externalGatewayMail, coreMail and platformBinding). The Services list, the Overview and the MCP tool cloudly_list_services read that one status, so a service that showed Failed or Converging for good because of a stage the Swarm-era Coreflow reconciler last reported on 2026-08-03 shows its real state. getServices composes the status for an administrator only; a cluster's relay gets the services without it. The Overview lists every failed service with its reason and a link to it (degraded ones as warnings); its Deployments tile and Recent deployment changes table are gone, and so is the dashboard summary's deployments count. Cloudly no longer serves reportReconciliationStatus and reportDeploymentSnapshots, the Swarm-era Coreflow intake no component sends, and an observer deletes its record when what it observes goes: a route withdrawn from a service and a deleted or tombstoned platform binding withdraw their externalGateway and platformBinding record. The data step deletes, through the database driver before any model is prepared, every reconciliation record of a kind only the Swarm-era Coreflow reconciler wrote (workload, routing, immutable-image, immutable-adoption and immutable-attestation) and every externalGateway record of a route its service no longer declares; a record of a route still declared stays, failed or not, so do the records of other observers, and a rerun removes nothing. The deployment operations get the index service_deployment_operation_newest, which the model builds at start. Cloudly 33.23.0 adds delegated admin temp tokens: a human administrator who signed in within five minutes mints a token for the scopes secrets and serviceCredentials that lives one hour at most (servezone admin temp-token mint, list and revoke); Cloudly stores only its SHA-256, in cloudly_delegated_admin_tokens, records every use, refusal and failed write in cloudly_delegated_admin_token_audit, and a token never writes a reserved name and never deletes. Cloudly 33.23.0 moves to @serve.zone/interfaces 32.61.0 and its dashboard to @design.estate/dees-catalog 23 (controls are 30px tall instead of 32px, dialogs and sheets have 14px corners instead of 20px). With Cloudly 33.22.0 the scheduled backup-all-services task, which 1.29.8 turned on with CLOUDLY_BACKUP_CRON, backs up each workload service on the node and cluster its data lives on: on a Cloudly with more than one cluster every service was refused, and on one with a single cluster every backup was refused backup-node-required, while the run was recorded completed. A service with nothing to back up is skipped, a service without a target fails by name, and a run in which any service failed ends failed and names each one. Cloudly 33.22.0 adds servezone service credential inspect, ensure, rotate and revoke for the service machine credential @serve.zone/platformclient 4.0.0 and later signs in with, and names a refused actor SERVICE_CREDENTIAL_UNAUTHORIZED. Neither release adds an environment variable or a setting, and neither changes the Dockerfile or the qenv.yml requirements, so 1.29.9 keeps 1.29.8's runtime contract, CLOUDLY_BACKUP_CRON 15 2 * * * included, and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: the data step has no way back. Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.29.8, 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. 1.29.9 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.9 still exports the ledger with its documents; 1.30.5 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.9 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.5, and 1.29.9 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2, 1.30.3 or 1.30.4, which sort above it. Like 1.29.8, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.8 the upgrade moves Cloudly from 33.21.4 to 33.23.0 and runs the data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0. From 1.29.7 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.8, the Cloudly 33.21.4 template, and describes the changes of Cloudly 33.21.4 and the releases before it that an upgrade from 1.29.7 or earlier also takes: Move Cloudly to 33.21.4, a fixes-only patch over 33.21.3, on the config of 1.29.7, and turn on Cloudly's scheduled workload backups: 1.29.8 differs from 1.29.7 only in its image, its changelog and the environment variable CLOUDLY_BACKUP_CRON. Cloudly registers its backup-all-services task only when CLOUDLY_BACKUP_CRON is set, and no earlier template set it, so the databases of the workload services Cloudly runs were never backed up on a schedule. This template sets it to 15 2 * * *: every day at 02:15 UTC, clear of Onebox's backup-archive-prune at 03:00, Cloudly creates a backup of every workload service with backup-enabled resources, and it keeps the newest 24 settled backups of each service (CLOUDLY_BACKUP_KEEP_LAST, which the template does not set, defaults to 24) and deletes older ones with their stored archives. These backups cover the workload databases; a workload's object storage is not part of them (AGL-473). The variable is a plain declaration with no secret and no delivery and is not required. A host that upgrades a service gives a key the service does not have yet the template's value, so an upgraded Cloudly runs the schedule; an operator's stored value for the key wins over the template's. Cloudly 33.21.4 derives the owner of a launcher-environment secret file from the identity the workload runs as (AGL-459): every launcher delivery was published 0:0 and 0400, so a workload that runs as another user could not read its secrets and WorkloadInit exited with SECRET_SOURCE. The manifest publication now derives the owner per target cluster from the spec's runAs and the pinned image's config.User with @serve.zone/interfaces 32.60.0 deriveLauncherSecretDeliveryOwner, an owner that does not derive is refused launcher-secret-ownership-unknown, and the manifests 33.21.3 published are published again on upgrade. On @serve.zone/interfaces 32.60.0, external registry answers no longer carry the stored credential, Test connection proves the credentials through the Docker Registry v2 login, and a service update merges only the fields it states, so Edit and Configure Mail work again for a service with a runtime spec or an immutable image. Cloudly 33.21.4 also fixes the dashboard (AGL-387 to AGL-401: the status bar, the fonts, the dropdown presets, the view actions, the Secrets wizard and index, the Logs view and the App Store view), refuses a deployment without an existing service and node, a node action the node can never receive and a Jump code for a cluster without a relay, and bootstraps a node with the pinned Spark 35.1.18 release. Cloudly 33.21.4 adds no data step and keeps the data version 0.32.0, so an upgrade from 1.29.7 runs no data step; it adds no index, no collection and no setting, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is that of 1.29.7 with CLOUDLY_BACKUP_CRON and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and an upgrade from 1.29.6 or earlier runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0, which has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. 1.29.8 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.8 still exports the ledger with its documents; 1.30.4 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.8 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.4, and 1.29.8 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2 or 1.30.3, which sort above it. Like 1.29.7, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.7 the upgrade moves Cloudly from 33.21.3 to 33.21.4 and turns the scheduled workload backups on. From 1.29.6 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.7, the Cloudly 33.21.3 template, and describes the changes of Cloudly 33.21.3 and the releases before it that an upgrade from 1.29.6 or earlier also takes: Move Cloudly to 33.21.3, a patch over 33.21.2 that adds one forward-only data step, on the config of 1.29.6: 1.29.7 differs from 1.29.6 only in its image and changelog. A service migrated from Swarm reaches its object storage at its node's Corestore endpoint again. The Secrets v2 cutover turned the Docker-era Corestore binding environment, the nine object-storage endpoint names and, for a binding with retention, the eleven S3_RETENTION_* names, into service-owned secrets in scope platform:cloudly-corestore, the endpoint names with the Docker host corestore; a Corestore binding now gives the endpoint names as plain environment composed from the node's endpoint, and the secrets the cutover left behind replaced that environment in the workload, since a secret wins over an inherited variable of the same name, so such a service exited at startup with getaddrinfo ENOTFOUND corestore. Cloudly 33.21.3 adds the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. It retires every active service-owned secret in that scope whose key is one of the twenty non-secret object-storage names (the nine endpoint names and the eleven S3_RETENTION_* names) as a lifecycle change retires one: the version is retired and kept, never purged, and the metadata names the step as the author. The service's secrets revision advances once, and its desired manifests are published again without those secrets under a new secret rollout, the previous one accepted for five more minutes, so the runtime replaces the service's replicas once; a single-replica service restarts briefly. A service whose image plan or secret deployment plan the runtime already refuses gets its manifest from its next promotion. The six credential names, other scopes, other owners and greenfield services are left alone; a database binding gives no plain name, so no database secret is touched. Each service is one transaction, so a rerun writes nothing. The step has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. Cloudly 33.21.3 adds no index, no collection, no setting and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.6 and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and every upgrade to this template runs the step to 0.32.0. 1.29.7 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.7 still exports the ledger with its documents; 1.30.3 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.7 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.3, and 1.29.7 is never the upgrade target of 1.30.0, 1.30.1 or 1.30.2, which sort above it. Like 1.29.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.6 the upgrade moves Cloudly from 33.21.2 to 33.21.3 and runs the step to 0.32.0. From 1.29.5 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.6, the Cloudly 33.21.2 template, and describes the changes of Cloudly 33.21.2 and the releases before it that an upgrade from 1.29.5 or earlier also takes: Move Cloudly to 33.21.2, a fixes-only patch over 33.21.1, on the config of 1.29.5: 1.29.6 differs from 1.29.5 only in its image and changelog. A recovery redeploy over a dead retained rollout plan, one that no runtime can execute (such as a Swarm-era plan) and whose operation is settled, is no longer refused PLATFORM_CAPABILITY_UNAVAILABLE by its reserve and promote preflights while its Corestore bindings wait for their credentials: such a service has no plan a runtime can run, so it is issued no credential grant until a promotion writes one, and its database and object-storage bindings could not become ready before the promotion those preflights refused. The reserve and promote preflights now defer the readiness of such a binding while it is requested or provisioning, as the promote preflight of a held service's first promotion already did, and warn PLATFORM_BINDING_READINESS_DEFERRED; a binding that adopts a legacy resource is deferred only while Corestore lists that resource as authorized for it, and the route preflight after the rollout still requires every binding ready. The retained plan is judged by the operation it names, as the promotion fence reads it, never by the service's latest operation. 1.29.6 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.6 still exports the ledger with its documents; 1.30.2 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.6 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.2, and 1.29.6 is never the upgrade target of 1.30.0 or 1.30.1, which sort above it. Cloudly 33.21.2 adds no data step, no index and no setting and keeps the data version 0.31.0, so an upgrade from 1.29.5, 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.5 and requires Onebox 33.8.0. Like 1.29.5, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.5 the upgrade moves Cloudly from 33.21.1 to 33.21.2. From 1.29.4 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5, the Cloudly 33.21.1 template, and describes the changes of Cloudly 33.21.1 and the releases before it that an upgrade from 1.29.4 or earlier also takes: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
               "breaking": true,
               "requiresManualReview": true,
               "migrationRequired": true,
@@ -11237,12 +12073,12 @@
                 "expectedStatus": 200
               }
             },
-            "configDigest": "sha256:61778445373bd108eac47584385e2a82558cb742942f0d96d065554c8ad1ac4b"
+            "configDigest": "sha256:0c8c9268d7556b65c0340c3c2f8b9813001bdd86441c01708531fb46ee4a46e3"
           },
           {
-            "version": "1.30.5",
+            "version": "1.30.7",
             "config": {
-              "image": "code.foss.global/serve.zone/cloudly@sha256:73be2fb57ee6bf40f83569c677f381713a7ddcc38fedbb3f16a7d85fb1e6e33d",
+              "image": "code.foss.global/serve.zone/cloudly@sha256:e230729d762f770d20643e05d769d92ad27038dc64cb9fac448f45b2f881b43a",
               "targetPorts": [
                 {
                   "name": "web",
@@ -11432,7 +12268,7 @@
                 "clientAuthenticationMethod": "client_secret_basic"
               },
               "minOneboxVersion": "33.8.0",
-              "changelog": "Declare Cloudly's runtime log ledger non-backup data without moving the image: this template runs the Cloudly 33.23.0 image of 1.29.9 and differs from 1.29.9 only in its changelog and two members. databaseBackupExclusions names the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. Every Cloudly start writes the log record of each running attempt that has none before it serves node requests, so the nodes' next batches are kept again after a restore, and the first batch of each capture states the numbers the restore lost as a gap. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.9; a host without the object-storage feature keeps 1.28.4. Like 1.29.9, this template sets CLOUDLY_BACKUP_CRON to 15 2 * * *, so Cloudly backs up the databases of its workload services daily at 02:15 UTC (not their object storage, AGL-473). From 1.29.9 the upgrade moves the template only: Cloudly runs the same image on the same data and no data step runs. From 1.30.4 it moves Cloudly from 33.21.4 to 33.23.0, keeps the declaration and runs the forward-only data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0. From 1.30.3 it moves Cloudly from 33.21.3 to 33.23.0, keeps the declaration, turns the scheduled workload backups on and runs that step. From 1.30.2 it moves Cloudly from 33.21.2 to 33.23.0, keeps the declaration, turns them on and runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0 and then that step. From 1.30.1 it moves Cloudly from 33.21.1 to 33.23.0 and from 1.30.0 from 33.21.0, keeps the declaration, turns them on and runs both steps. Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. From 1.29.8 or earlier the template also gains the declaration and the upgrade takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.9 and holds for this template: Move Cloudly to 33.23.0 on the config of 1.29.8: 1.29.9 differs from 1.29.8 only in its image and changelog. Cloudly 33.23.0 adds one forward-only data step, retire-stale-reconciliation-stages from 0.32.0 to 0.33.0, and takes the changes of Cloudly 33.22.0, which no template ran. A service's status is now what the producers Cloudly runs state about it: the Pallet runtime's verdict on each target, the current plan's rollout, the newest deployment operation and the records of the observers Cloudly runs (externalGateway, externalGatewayMail, coreMail and platformBinding). The Services list, the Overview and the MCP tool cloudly_list_services read that one status, so a service that showed Failed or Converging for good because of a stage the Swarm-era Coreflow reconciler last reported on 2026-08-03 shows its real state. getServices composes the status for an administrator only; a cluster's relay gets the services without it. The Overview lists every failed service with its reason and a link to it (degraded ones as warnings); its Deployments tile and Recent deployment changes table are gone, and so is the dashboard summary's deployments count. Cloudly no longer serves reportReconciliationStatus and reportDeploymentSnapshots, the Swarm-era Coreflow intake no component sends, and an observer deletes its record when what it observes goes: a route withdrawn from a service and a deleted or tombstoned platform binding withdraw their externalGateway and platformBinding record. The data step deletes, through the database driver before any model is prepared, every reconciliation record of a kind only the Swarm-era Coreflow reconciler wrote (workload, routing, immutable-image, immutable-adoption and immutable-attestation) and every externalGateway record of a route its service no longer declares; a record of a route still declared stays, failed or not, so do the records of other observers, and a rerun removes nothing. The deployment operations get the index service_deployment_operation_newest, which the model builds at start. Cloudly 33.23.0 adds delegated admin temp tokens: a human administrator who signed in within five minutes mints a token for the scopes secrets and serviceCredentials that lives one hour at most (servezone admin temp-token mint, list and revoke); Cloudly stores only its SHA-256, in cloudly_delegated_admin_tokens, records every use, refusal and failed write in cloudly_delegated_admin_token_audit, and a token never writes a reserved name and never deletes. Cloudly 33.23.0 moves to @serve.zone/interfaces 32.61.0 and its dashboard to @design.estate/dees-catalog 23 (controls are 30px tall instead of 32px, dialogs and sheets have 14px corners instead of 20px). With Cloudly 33.22.0 the scheduled backup-all-services task, which 1.29.8 turned on with CLOUDLY_BACKUP_CRON, backs up each workload service on the node and cluster its data lives on: on a Cloudly with more than one cluster every service was refused, and on one with a single cluster every backup was refused backup-node-required, while the run was recorded completed. A service with nothing to back up is skipped, a service without a target fails by name, and a run in which any service failed ends failed and names each one. Cloudly 33.22.0 adds servezone service credential inspect, ensure, rotate and revoke for the service machine credential @serve.zone/platformclient 4.0.0 and later signs in with, and names a refused actor SERVICE_CREDENTIAL_UNAUTHORIZED. Neither release adds an environment variable or a setting, and neither changes the Dockerfile or the qenv.yml requirements, so 1.29.9 keeps 1.29.8's runtime contract, CLOUDLY_BACKUP_CRON 15 2 * * * included, and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: the data step has no way back. Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.29.8, 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. 1.29.9 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.9 still exports the ledger with its documents; 1.30.5 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.9 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.5, and 1.29.9 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2, 1.30.3 or 1.30.4, which sort above it. Like 1.29.8, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.8 the upgrade moves Cloudly from 33.21.4 to 33.23.0 and runs the data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0. From 1.29.7 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.8, the Cloudly 33.21.4 template, and describes the changes of Cloudly 33.21.4 and the releases before it that an upgrade from 1.29.7 or earlier also takes: Move Cloudly to 33.21.4, a fixes-only patch over 33.21.3, on the config of 1.29.7, and turn on Cloudly's scheduled workload backups: 1.29.8 differs from 1.29.7 only in its image, its changelog and the environment variable CLOUDLY_BACKUP_CRON. Cloudly registers its backup-all-services task only when CLOUDLY_BACKUP_CRON is set, and no earlier template set it, so the databases of the workload services Cloudly runs were never backed up on a schedule. This template sets it to 15 2 * * *: every day at 02:15 UTC, clear of Onebox's backup-archive-prune at 03:00, Cloudly creates a backup of every workload service with backup-enabled resources, and it keeps the newest 24 settled backups of each service (CLOUDLY_BACKUP_KEEP_LAST, which the template does not set, defaults to 24) and deletes older ones with their stored archives. These backups cover the workload databases; a workload's object storage is not part of them (AGL-473). The variable is a plain declaration with no secret and no delivery and is not required. A host that upgrades a service gives a key the service does not have yet the template's value, so an upgraded Cloudly runs the schedule; an operator's stored value for the key wins over the template's. Cloudly 33.21.4 derives the owner of a launcher-environment secret file from the identity the workload runs as (AGL-459): every launcher delivery was published 0:0 and 0400, so a workload that runs as another user could not read its secrets and WorkloadInit exited with SECRET_SOURCE. The manifest publication now derives the owner per target cluster from the spec's runAs and the pinned image's config.User with @serve.zone/interfaces 32.60.0 deriveLauncherSecretDeliveryOwner, an owner that does not derive is refused launcher-secret-ownership-unknown, and the manifests 33.21.3 published are published again on upgrade. On @serve.zone/interfaces 32.60.0, external registry answers no longer carry the stored credential, Test connection proves the credentials through the Docker Registry v2 login, and a service update merges only the fields it states, so Edit and Configure Mail work again for a service with a runtime spec or an immutable image. Cloudly 33.21.4 also fixes the dashboard (AGL-387 to AGL-401: the status bar, the fonts, the dropdown presets, the view actions, the Secrets wizard and index, the Logs view and the App Store view), refuses a deployment without an existing service and node, a node action the node can never receive and a Jump code for a cluster without a relay, and bootstraps a node with the pinned Spark 35.1.18 release. Cloudly 33.21.4 adds no data step and keeps the data version 0.32.0, so an upgrade from 1.29.7 runs no data step; it adds no index, no collection and no setting, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is that of 1.29.7 with CLOUDLY_BACKUP_CRON and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and an upgrade from 1.29.6 or earlier runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0, which has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. 1.29.8 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.8 still exports the ledger with its documents; 1.30.4 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.8 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.4, and 1.29.8 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2 or 1.30.3, which sort above it. Like 1.29.7, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.7 the upgrade moves Cloudly from 33.21.3 to 33.21.4 and turns the scheduled workload backups on. From 1.29.6 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.7, the Cloudly 33.21.3 template, and describes the changes of Cloudly 33.21.3 and the releases before it that an upgrade from 1.29.6 or earlier also takes: Move Cloudly to 33.21.3, a patch over 33.21.2 that adds one forward-only data step, on the config of 1.29.6: 1.29.7 differs from 1.29.6 only in its image and changelog. A service migrated from Swarm reaches its object storage at its node's Corestore endpoint again. The Secrets v2 cutover turned the Docker-era Corestore binding environment, the nine object-storage endpoint names and, for a binding with retention, the eleven S3_RETENTION_* names, into service-owned secrets in scope platform:cloudly-corestore, the endpoint names with the Docker host corestore; a Corestore binding now gives the endpoint names as plain environment composed from the node's endpoint, and the secrets the cutover left behind replaced that environment in the workload, since a secret wins over an inherited variable of the same name, so such a service exited at startup with getaddrinfo ENOTFOUND corestore. Cloudly 33.21.3 adds the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. It retires every active service-owned secret in that scope whose key is one of the twenty non-secret object-storage names (the nine endpoint names and the eleven S3_RETENTION_* names) as a lifecycle change retires one: the version is retired and kept, never purged, and the metadata names the step as the author. The service's secrets revision advances once, and its desired manifests are published again without those secrets under a new secret rollout, the previous one accepted for five more minutes, so the runtime replaces the service's replicas once; a single-replica service restarts briefly. A service whose image plan or secret deployment plan the runtime already refuses gets its manifest from its next promotion. The six credential names, other scopes, other owners and greenfield services are left alone; a database binding gives no plain name, so no database secret is touched. Each service is one transaction, so a rerun writes nothing. The step has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. Cloudly 33.21.3 adds no index, no collection, no setting and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.6 and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and every upgrade to this template runs the step to 0.32.0. 1.29.7 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.7 still exports the ledger with its documents; 1.30.3 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.7 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.3, and 1.29.7 is never the upgrade target of 1.30.0, 1.30.1 or 1.30.2, which sort above it. Like 1.29.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.6 the upgrade moves Cloudly from 33.21.2 to 33.21.3 and runs the step to 0.32.0. From 1.29.5 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.6, the Cloudly 33.21.2 template, and describes the changes of Cloudly 33.21.2 and the releases before it that an upgrade from 1.29.5 or earlier also takes: Move Cloudly to 33.21.2, a fixes-only patch over 33.21.1, on the config of 1.29.5: 1.29.6 differs from 1.29.5 only in its image and changelog. A recovery redeploy over a dead retained rollout plan, one that no runtime can execute (such as a Swarm-era plan) and whose operation is settled, is no longer refused PLATFORM_CAPABILITY_UNAVAILABLE by its reserve and promote preflights while its Corestore bindings wait for their credentials: such a service has no plan a runtime can run, so it is issued no credential grant until a promotion writes one, and its database and object-storage bindings could not become ready before the promotion those preflights refused. The reserve and promote preflights now defer the readiness of such a binding while it is requested or provisioning, as the promote preflight of a held service's first promotion already did, and warn PLATFORM_BINDING_READINESS_DEFERRED; a binding that adopts a legacy resource is deferred only while Corestore lists that resource as authorized for it, and the route preflight after the rollout still requires every binding ready. The retained plan is judged by the operation it names, as the promotion fence reads it, never by the service's latest operation. 1.29.6 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.6 still exports the ledger with its documents; 1.30.2 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.6 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.2, and 1.29.6 is never the upgrade target of 1.30.0 or 1.30.1, which sort above it. Cloudly 33.21.2 adds no data step, no index and no setting and keeps the data version 0.31.0, so an upgrade from 1.29.5, 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.5 and requires Onebox 33.8.0. Like 1.29.5, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.5 the upgrade moves Cloudly from 33.21.1 to 33.21.2. From 1.29.4 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5, the Cloudly 33.21.1 template, and describes the changes of Cloudly 33.21.1 and the releases before it that an upgrade from 1.29.4 or earlier also takes: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
+              "changelog": "Declare Cloudly's runtime log ledger non-backup data without moving the image: this template runs the Cloudly 33.24.0 image of 1.29.11 and differs from 1.29.11 only in its changelog and two members. databaseBackupExclusions names the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. Every Cloudly start writes the log record of each running attempt that has none before it serves node requests, so the nodes' next batches are kept again after a restore, and the first batch of each capture states the numbers the restore lost as a gap. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.11; a host without the object-storage feature keeps 1.28.4. Like 1.29.11, this template sets CLOUDLY_BACKUP_CRON to 15 2 * * *, so Cloudly backs up the databases of its workload services daily at 02:15 UTC (not their object storage, AGL-473). From 1.29.11 the upgrade moves the template only: Cloudly runs the same image on the same data and no data step runs. From 1.30.6 it moves Cloudly from 33.23.2 to 33.24.0, keeps the declaration and runs no data step. From 1.30.5 it moves Cloudly from 33.23.0 to 33.24.0, keeps the declaration and runs no data step. From 1.30.4 it moves Cloudly from 33.21.4 to 33.24.0, keeps the declaration and runs the forward-only data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0. From 1.30.3 it moves Cloudly from 33.21.3 to 33.24.0, keeps the declaration, turns the scheduled workload backups on and runs that step. From 1.30.2 it moves Cloudly from 33.21.2 to 33.24.0, keeps the declaration, turns them on and runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0 and then that step. From 1.30.1 it moves Cloudly from 33.21.1 to 33.24.0 and from 1.30.0 from 33.21.0, keeps the declaration, turns them on and runs both steps. Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. From 1.29.10 or earlier the template also gains the declaration and the upgrade takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.11 and holds for this template: Move Cloudly to 33.24.0 on the config of 1.29.10: 1.29.11 differs from 1.29.10 only in its image and changelog. Cloudly 33.24.0 adds Temp tokens to the dashboard: Security, Temp tokens (/temp-tokens) mints, lists and revokes delegated admin temp tokens, so an administrator who signs in with a passkey through the platform provider (Onebox) mints one without a Cloudly password. A mint states its scopes (Secrets, Service credentials), its secret key prefixes (only with Secrets, at most 16) and a lifetime of 1 to 60 minutes. The token is shown once, in the answer's dialog, which drops it when it closes; the dashboard never writes it to a state part, browser storage, the URL or a log. Minting still needs a sign-in within the last five minutes, the rule Cloudly already applied and keeps unchanged: the page says until when the session can mint and offers Sign in again outside that window, and a provider session older than five minutes is signed into again at the provider first. The list shows the active tokens newest first, and Revoke asks for confirmation first. The release adds no data step, no environment variable, no setting, no collection and no index and changes neither the Dockerfile nor the qenv.yml requirements, so 1.29.11 keeps 1.29.10's runtime contract, CLOUDLY_BACKUP_CRON 15 2 * * * included, the data version 0.33.0 and the Onebox minimum 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and an upgrade from 1.29.8 or earlier runs the forward-only data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0, which has no way back: Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.29.8, 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. 1.29.11 does not declare the runtime log ledger databaseBackupExclusions; 1.30.7 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.11 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.7, and 1.29.11 is never the upgrade target of 1.30.0 to 1.30.6, which sort above it. Like 1.29.10, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.10 the upgrade moves Cloudly from 33.23.2 to 33.24.0 and runs no data step. From 1.29.9 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.10, the Cloudly 33.23.2 template, and describes the changes of Cloudly 33.23.2 and the releases before it that an upgrade from 1.29.9 or earlier also takes: Move Cloudly to 33.23.2, a fixes-only patch over 33.23.0, on the config of 1.29.9: 1.29.10 differs from 1.29.9 only in its image and changelog. Cloudly 33.23.2 takes the changes of Cloudly 33.23.1, which no template ran. Cloudly 33.23.1 composes each service's status on its own: a service whose stored image deployment plan, rollout status or newest deployment operation violates the @serve.zone/interfaces contract in a field the status reads gets the status failed with one composition stage that names the violation, for example: Cloudly cannot compose this service's status: its stored image deployment plan violates the contract (releaseTag, operationId missing). A composition that throws gives that service the same failed status with the reason and one error log line naming the service, and every other service keeps its status. Cloudly 33.23.0 answered the administrator's services list, getServiceById of such a service and the MCP tool cloudly_list_services with an internal error when one service stored a plan written before the plan carried releaseTag and operationId. The stored documents are not changed; a new promotion replaces such a plan. Cloudly 33.23.2 lets servezone service credential ensure give a service whose SERVEZONE_PLATFORM_AUTHORIZATION an operator stored as an ordinary secret its service machine credential: ensure, run by a human administrator who signed in within five minutes or with a serviceCredentials temp token, takes that operator-managed row over in the same transaction, active, retired, revoked or deleted, revokes every legacy version that could still be issued, keeps every version, moves the row to the credential's platform scope stamped with the administrator and the token, records the takeover on the new version, publishes the service's secret manifests again and is fenced on the operator authority and revision it read. Before, ensure was refused SERVICE_CREDENTIAL_CONFLICT for such a service whatever an administrator did first. A secret of the key under any other authority is still refused, and the legacy user token itself is not revoked automatically. Neither release adds a data step, an environment variable, a setting, a collection or an index, and neither changes the Dockerfile or the qenv.yml requirements, so 1.29.10 keeps 1.29.9's runtime contract, CLOUDLY_BACKUP_CRON 15 2 * * * included, the data version 0.33.0 and the Onebox minimum 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and an upgrade from 1.29.8 or earlier runs the forward-only data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0, which has no way back: Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.29.8, 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. 1.29.10 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.10 still exports the ledger with its documents; 1.30.6 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.10 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.6, and 1.29.10 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2, 1.30.3, 1.30.4 or 1.30.5, which sort above it. Like 1.29.9, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.9 the upgrade moves Cloudly from 33.23.0 to 33.23.2 and runs no data step. From 1.29.8 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.9, the Cloudly 33.23.0 template, and describes the changes of Cloudly 33.23.0 and the releases before it that an upgrade from 1.29.8 or earlier also takes: Move Cloudly to 33.23.0 on the config of 1.29.8: 1.29.9 differs from 1.29.8 only in its image and changelog. Cloudly 33.23.0 adds one forward-only data step, retire-stale-reconciliation-stages from 0.32.0 to 0.33.0, and takes the changes of Cloudly 33.22.0, which no template ran. A service's status is now what the producers Cloudly runs state about it: the Pallet runtime's verdict on each target, the current plan's rollout, the newest deployment operation and the records of the observers Cloudly runs (externalGateway, externalGatewayMail, coreMail and platformBinding). The Services list, the Overview and the MCP tool cloudly_list_services read that one status, so a service that showed Failed or Converging for good because of a stage the Swarm-era Coreflow reconciler last reported on 2026-08-03 shows its real state. getServices composes the status for an administrator only; a cluster's relay gets the services without it. The Overview lists every failed service with its reason and a link to it (degraded ones as warnings); its Deployments tile and Recent deployment changes table are gone, and so is the dashboard summary's deployments count. Cloudly no longer serves reportReconciliationStatus and reportDeploymentSnapshots, the Swarm-era Coreflow intake no component sends, and an observer deletes its record when what it observes goes: a route withdrawn from a service and a deleted or tombstoned platform binding withdraw their externalGateway and platformBinding record. The data step deletes, through the database driver before any model is prepared, every reconciliation record of a kind only the Swarm-era Coreflow reconciler wrote (workload, routing, immutable-image, immutable-adoption and immutable-attestation) and every externalGateway record of a route its service no longer declares; a record of a route still declared stays, failed or not, so do the records of other observers, and a rerun removes nothing. The deployment operations get the index service_deployment_operation_newest, which the model builds at start. Cloudly 33.23.0 adds delegated admin temp tokens: a human administrator who signed in within five minutes mints a token for the scopes secrets and serviceCredentials that lives one hour at most (servezone admin temp-token mint, list and revoke); Cloudly stores only its SHA-256, in cloudly_delegated_admin_tokens, records every use, refusal and failed write in cloudly_delegated_admin_token_audit, and a token never writes a reserved name and never deletes. Cloudly 33.23.0 moves to @serve.zone/interfaces 32.61.0 and its dashboard to @design.estate/dees-catalog 23 (controls are 30px tall instead of 32px, dialogs and sheets have 14px corners instead of 20px). With Cloudly 33.22.0 the scheduled backup-all-services task, which 1.29.8 turned on with CLOUDLY_BACKUP_CRON, backs up each workload service on the node and cluster its data lives on: on a Cloudly with more than one cluster every service was refused, and on one with a single cluster every backup was refused backup-node-required, while the run was recorded completed. A service with nothing to back up is skipped, a service without a target fails by name, and a run in which any service failed ends failed and names each one. Cloudly 33.22.0 adds servezone service credential inspect, ensure, rotate and revoke for the service machine credential @serve.zone/platformclient 4.0.0 and later signs in with, and names a refused actor SERVICE_CREDENTIAL_UNAUTHORIZED. Neither release adds an environment variable or a setting, and neither changes the Dockerfile or the qenv.yml requirements, so 1.29.9 keeps 1.29.8's runtime contract, CLOUDLY_BACKUP_CRON 15 2 * * * included, and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: the data step has no way back. Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.29.8, 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. 1.29.9 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.9 still exports the ledger with its documents; 1.30.5 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.9 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.5, and 1.29.9 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2, 1.30.3 or 1.30.4, which sort above it. Like 1.29.8, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.8 the upgrade moves Cloudly from 33.21.4 to 33.23.0 and runs the data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0. From 1.29.7 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.8, the Cloudly 33.21.4 template, and describes the changes of Cloudly 33.21.4 and the releases before it that an upgrade from 1.29.7 or earlier also takes: Move Cloudly to 33.21.4, a fixes-only patch over 33.21.3, on the config of 1.29.7, and turn on Cloudly's scheduled workload backups: 1.29.8 differs from 1.29.7 only in its image, its changelog and the environment variable CLOUDLY_BACKUP_CRON. Cloudly registers its backup-all-services task only when CLOUDLY_BACKUP_CRON is set, and no earlier template set it, so the databases of the workload services Cloudly runs were never backed up on a schedule. This template sets it to 15 2 * * *: every day at 02:15 UTC, clear of Onebox's backup-archive-prune at 03:00, Cloudly creates a backup of every workload service with backup-enabled resources, and it keeps the newest 24 settled backups of each service (CLOUDLY_BACKUP_KEEP_LAST, which the template does not set, defaults to 24) and deletes older ones with their stored archives. These backups cover the workload databases; a workload's object storage is not part of them (AGL-473). The variable is a plain declaration with no secret and no delivery and is not required. A host that upgrades a service gives a key the service does not have yet the template's value, so an upgraded Cloudly runs the schedule; an operator's stored value for the key wins over the template's. Cloudly 33.21.4 derives the owner of a launcher-environment secret file from the identity the workload runs as (AGL-459): every launcher delivery was published 0:0 and 0400, so a workload that runs as another user could not read its secrets and WorkloadInit exited with SECRET_SOURCE. The manifest publication now derives the owner per target cluster from the spec's runAs and the pinned image's config.User with @serve.zone/interfaces 32.60.0 deriveLauncherSecretDeliveryOwner, an owner that does not derive is refused launcher-secret-ownership-unknown, and the manifests 33.21.3 published are published again on upgrade. On @serve.zone/interfaces 32.60.0, external registry answers no longer carry the stored credential, Test connection proves the credentials through the Docker Registry v2 login, and a service update merges only the fields it states, so Edit and Configure Mail work again for a service with a runtime spec or an immutable image. Cloudly 33.21.4 also fixes the dashboard (AGL-387 to AGL-401: the status bar, the fonts, the dropdown presets, the view actions, the Secrets wizard and index, the Logs view and the App Store view), refuses a deployment without an existing service and node, a node action the node can never receive and a Jump code for a cluster without a relay, and bootstraps a node with the pinned Spark 35.1.18 release. Cloudly 33.21.4 adds no data step and keeps the data version 0.32.0, so an upgrade from 1.29.7 runs no data step; it adds no index, no collection and no setting, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is that of 1.29.7 with CLOUDLY_BACKUP_CRON and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and an upgrade from 1.29.6 or earlier runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0, which has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. 1.29.8 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.8 still exports the ledger with its documents; 1.30.4 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.8 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.4, and 1.29.8 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2 or 1.30.3, which sort above it. Like 1.29.7, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.7 the upgrade moves Cloudly from 33.21.3 to 33.21.4 and turns the scheduled workload backups on. From 1.29.6 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.7, the Cloudly 33.21.3 template, and describes the changes of Cloudly 33.21.3 and the releases before it that an upgrade from 1.29.6 or earlier also takes: Move Cloudly to 33.21.3, a patch over 33.21.2 that adds one forward-only data step, on the config of 1.29.6: 1.29.7 differs from 1.29.6 only in its image and changelog. A service migrated from Swarm reaches its object storage at its node's Corestore endpoint again. The Secrets v2 cutover turned the Docker-era Corestore binding environment, the nine object-storage endpoint names and, for a binding with retention, the eleven S3_RETENTION_* names, into service-owned secrets in scope platform:cloudly-corestore, the endpoint names with the Docker host corestore; a Corestore binding now gives the endpoint names as plain environment composed from the node's endpoint, and the secrets the cutover left behind replaced that environment in the workload, since a secret wins over an inherited variable of the same name, so such a service exited at startup with getaddrinfo ENOTFOUND corestore. Cloudly 33.21.3 adds the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. It retires every active service-owned secret in that scope whose key is one of the twenty non-secret object-storage names (the nine endpoint names and the eleven S3_RETENTION_* names) as a lifecycle change retires one: the version is retired and kept, never purged, and the metadata names the step as the author. The service's secrets revision advances once, and its desired manifests are published again without those secrets under a new secret rollout, the previous one accepted for five more minutes, so the runtime replaces the service's replicas once; a single-replica service restarts briefly. A service whose image plan or secret deployment plan the runtime already refuses gets its manifest from its next promotion. The six credential names, other scopes, other owners and greenfield services are left alone; a database binding gives no plain name, so no database secret is touched. Each service is one transaction, so a rerun writes nothing. The step has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. Cloudly 33.21.3 adds no index, no collection, no setting and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.6 and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and every upgrade to this template runs the step to 0.32.0. 1.29.7 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.7 still exports the ledger with its documents; 1.30.3 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.7 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.3, and 1.29.7 is never the upgrade target of 1.30.0, 1.30.1 or 1.30.2, which sort above it. Like 1.29.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.6 the upgrade moves Cloudly from 33.21.2 to 33.21.3 and runs the step to 0.32.0. From 1.29.5 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.6, the Cloudly 33.21.2 template, and describes the changes of Cloudly 33.21.2 and the releases before it that an upgrade from 1.29.5 or earlier also takes: Move Cloudly to 33.21.2, a fixes-only patch over 33.21.1, on the config of 1.29.5: 1.29.6 differs from 1.29.5 only in its image and changelog. A recovery redeploy over a dead retained rollout plan, one that no runtime can execute (such as a Swarm-era plan) and whose operation is settled, is no longer refused PLATFORM_CAPABILITY_UNAVAILABLE by its reserve and promote preflights while its Corestore bindings wait for their credentials: such a service has no plan a runtime can run, so it is issued no credential grant until a promotion writes one, and its database and object-storage bindings could not become ready before the promotion those preflights refused. The reserve and promote preflights now defer the readiness of such a binding while it is requested or provisioning, as the promote preflight of a held service's first promotion already did, and warn PLATFORM_BINDING_READINESS_DEFERRED; a binding that adopts a legacy resource is deferred only while Corestore lists that resource as authorized for it, and the route preflight after the rollout still requires every binding ready. The retained plan is judged by the operation it names, as the promotion fence reads it, never by the service's latest operation. 1.29.6 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.6 still exports the ledger with its documents; 1.30.2 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.6 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.2, and 1.29.6 is never the upgrade target of 1.30.0 or 1.30.1, which sort above it. Cloudly 33.21.2 adds no data step, no index and no setting and keeps the data version 0.31.0, so an upgrade from 1.29.5, 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.5 and requires Onebox 33.8.0. Like 1.29.5, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.5 the upgrade moves Cloudly from 33.21.1 to 33.21.2. From 1.29.4 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5, the Cloudly 33.21.1 template, and describes the changes of Cloudly 33.21.1 and the releases before it that an upgrade from 1.29.4 or earlier also takes: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
               "breaking": true,
               "requiresManualReview": true,
               "migrationRequired": true,
@@ -11447,12 +12283,12 @@
                 "expectedStatus": 200
               }
             },
-            "configDigest": "sha256:e30a131ead0cda7dbb4aa799a78ca991d75cf917d9d25abe7c02e604743af19a"
+            "configDigest": "sha256:cf68bdbc782b8739aa90067b048bea1cf2890b737d0cbb7f008414995b136cef"
           },
           {
-            "version": "1.30.6",
+            "version": "1.30.8",
             "config": {
-              "image": "code.foss.global/serve.zone/cloudly@sha256:cd0808e1fa4fdcf49742ac527480013ca506eb653fea4f82a86de546d1186876",
+              "image": "code.foss.global/serve.zone/cloudly@sha256:b5b1d9f0ba494d4e8472f141c03b7b76a8590d256debfd9c144f1eb866c18069",
               "targetPorts": [
                 {
                   "name": "web",
@@ -11642,7 +12478,7 @@
                 "clientAuthenticationMethod": "client_secret_basic"
               },
               "minOneboxVersion": "33.8.0",
-              "changelog": "Declare Cloudly's runtime log ledger non-backup data without moving the image: this template runs the Cloudly 33.23.2 image of 1.29.10 and differs from 1.29.10 only in its changelog and two members. databaseBackupExclusions names the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. Every Cloudly start writes the log record of each running attempt that has none before it serves node requests, so the nodes' next batches are kept again after a restore, and the first batch of each capture states the numbers the restore lost as a gap. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.10; a host without the object-storage feature keeps 1.28.4. Like 1.29.10, this template sets CLOUDLY_BACKUP_CRON to 15 2 * * *, so Cloudly backs up the databases of its workload services daily at 02:15 UTC (not their object storage, AGL-473). From 1.29.10 the upgrade moves the template only: Cloudly runs the same image on the same data and no data step runs. From 1.30.5 it moves Cloudly from 33.23.0 to 33.23.2, keeps the declaration and runs no data step. From 1.30.4 it moves Cloudly from 33.21.4 to 33.23.2, keeps the declaration and runs the forward-only data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0. From 1.30.3 it moves Cloudly from 33.21.3 to 33.23.2, keeps the declaration, turns the scheduled workload backups on and runs that step. From 1.30.2 it moves Cloudly from 33.21.2 to 33.23.2, keeps the declaration, turns them on and runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0 and then that step. From 1.30.1 it moves Cloudly from 33.21.1 to 33.23.2 and from 1.30.0 from 33.21.0, keeps the declaration, turns them on and runs both steps. Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. From 1.29.9 or earlier the template also gains the declaration and the upgrade takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.10 and holds for this template: Move Cloudly to 33.23.2, a fixes-only patch over 33.23.0, on the config of 1.29.9: 1.29.10 differs from 1.29.9 only in its image and changelog. Cloudly 33.23.2 takes the changes of Cloudly 33.23.1, which no template ran. Cloudly 33.23.1 composes each service's status on its own: a service whose stored image deployment plan, rollout status or newest deployment operation violates the @serve.zone/interfaces contract in a field the status reads gets the status failed with one composition stage that names the violation, for example: Cloudly cannot compose this service's status: its stored image deployment plan violates the contract (releaseTag, operationId missing). A composition that throws gives that service the same failed status with the reason and one error log line naming the service, and every other service keeps its status. Cloudly 33.23.0 answered the administrator's services list, getServiceById of such a service and the MCP tool cloudly_list_services with an internal error when one service stored a plan written before the plan carried releaseTag and operationId. The stored documents are not changed; a new promotion replaces such a plan. Cloudly 33.23.2 lets servezone service credential ensure give a service whose SERVEZONE_PLATFORM_AUTHORIZATION an operator stored as an ordinary secret its service machine credential: ensure, run by a human administrator who signed in within five minutes or with a serviceCredentials temp token, takes that operator-managed row over in the same transaction, active, retired, revoked or deleted, revokes every legacy version that could still be issued, keeps every version, moves the row to the credential's platform scope stamped with the administrator and the token, records the takeover on the new version, publishes the service's secret manifests again and is fenced on the operator authority and revision it read. Before, ensure was refused SERVICE_CREDENTIAL_CONFLICT for such a service whatever an administrator did first. A secret of the key under any other authority is still refused, and the legacy user token itself is not revoked automatically. Neither release adds a data step, an environment variable, a setting, a collection or an index, and neither changes the Dockerfile or the qenv.yml requirements, so 1.29.10 keeps 1.29.9's runtime contract, CLOUDLY_BACKUP_CRON 15 2 * * * included, the data version 0.33.0 and the Onebox minimum 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and an upgrade from 1.29.8 or earlier runs the forward-only data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0, which has no way back: Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.29.8, 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. 1.29.10 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.10 still exports the ledger with its documents; 1.30.6 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.10 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.6, and 1.29.10 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2, 1.30.3, 1.30.4 or 1.30.5, which sort above it. Like 1.29.9, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.9 the upgrade moves Cloudly from 33.23.0 to 33.23.2 and runs no data step. From 1.29.8 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.9, the Cloudly 33.23.0 template, and describes the changes of Cloudly 33.23.0 and the releases before it that an upgrade from 1.29.8 or earlier also takes: Move Cloudly to 33.23.0 on the config of 1.29.8: 1.29.9 differs from 1.29.8 only in its image and changelog. Cloudly 33.23.0 adds one forward-only data step, retire-stale-reconciliation-stages from 0.32.0 to 0.33.0, and takes the changes of Cloudly 33.22.0, which no template ran. A service's status is now what the producers Cloudly runs state about it: the Pallet runtime's verdict on each target, the current plan's rollout, the newest deployment operation and the records of the observers Cloudly runs (externalGateway, externalGatewayMail, coreMail and platformBinding). The Services list, the Overview and the MCP tool cloudly_list_services read that one status, so a service that showed Failed or Converging for good because of a stage the Swarm-era Coreflow reconciler last reported on 2026-08-03 shows its real state. getServices composes the status for an administrator only; a cluster's relay gets the services without it. The Overview lists every failed service with its reason and a link to it (degraded ones as warnings); its Deployments tile and Recent deployment changes table are gone, and so is the dashboard summary's deployments count. Cloudly no longer serves reportReconciliationStatus and reportDeploymentSnapshots, the Swarm-era Coreflow intake no component sends, and an observer deletes its record when what it observes goes: a route withdrawn from a service and a deleted or tombstoned platform binding withdraw their externalGateway and platformBinding record. The data step deletes, through the database driver before any model is prepared, every reconciliation record of a kind only the Swarm-era Coreflow reconciler wrote (workload, routing, immutable-image, immutable-adoption and immutable-attestation) and every externalGateway record of a route its service no longer declares; a record of a route still declared stays, failed or not, so do the records of other observers, and a rerun removes nothing. The deployment operations get the index service_deployment_operation_newest, which the model builds at start. Cloudly 33.23.0 adds delegated admin temp tokens: a human administrator who signed in within five minutes mints a token for the scopes secrets and serviceCredentials that lives one hour at most (servezone admin temp-token mint, list and revoke); Cloudly stores only its SHA-256, in cloudly_delegated_admin_tokens, records every use, refusal and failed write in cloudly_delegated_admin_token_audit, and a token never writes a reserved name and never deletes. Cloudly 33.23.0 moves to @serve.zone/interfaces 32.61.0 and its dashboard to @design.estate/dees-catalog 23 (controls are 30px tall instead of 32px, dialogs and sheets have 14px corners instead of 20px). With Cloudly 33.22.0 the scheduled backup-all-services task, which 1.29.8 turned on with CLOUDLY_BACKUP_CRON, backs up each workload service on the node and cluster its data lives on: on a Cloudly with more than one cluster every service was refused, and on one with a single cluster every backup was refused backup-node-required, while the run was recorded completed. A service with nothing to back up is skipped, a service without a target fails by name, and a run in which any service failed ends failed and names each one. Cloudly 33.22.0 adds servezone service credential inspect, ensure, rotate and revoke for the service machine credential @serve.zone/platformclient 4.0.0 and later signs in with, and names a refused actor SERVICE_CREDENTIAL_UNAUTHORIZED. Neither release adds an environment variable or a setting, and neither changes the Dockerfile or the qenv.yml requirements, so 1.29.9 keeps 1.29.8's runtime contract, CLOUDLY_BACKUP_CRON 15 2 * * * included, and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: the data step has no way back. Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.29.8, 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. 1.29.9 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.9 still exports the ledger with its documents; 1.30.5 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.9 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.5, and 1.29.9 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2, 1.30.3 or 1.30.4, which sort above it. Like 1.29.8, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.8 the upgrade moves Cloudly from 33.21.4 to 33.23.0 and runs the data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0. From 1.29.7 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.8, the Cloudly 33.21.4 template, and describes the changes of Cloudly 33.21.4 and the releases before it that an upgrade from 1.29.7 or earlier also takes: Move Cloudly to 33.21.4, a fixes-only patch over 33.21.3, on the config of 1.29.7, and turn on Cloudly's scheduled workload backups: 1.29.8 differs from 1.29.7 only in its image, its changelog and the environment variable CLOUDLY_BACKUP_CRON. Cloudly registers its backup-all-services task only when CLOUDLY_BACKUP_CRON is set, and no earlier template set it, so the databases of the workload services Cloudly runs were never backed up on a schedule. This template sets it to 15 2 * * *: every day at 02:15 UTC, clear of Onebox's backup-archive-prune at 03:00, Cloudly creates a backup of every workload service with backup-enabled resources, and it keeps the newest 24 settled backups of each service (CLOUDLY_BACKUP_KEEP_LAST, which the template does not set, defaults to 24) and deletes older ones with their stored archives. These backups cover the workload databases; a workload's object storage is not part of them (AGL-473). The variable is a plain declaration with no secret and no delivery and is not required. A host that upgrades a service gives a key the service does not have yet the template's value, so an upgraded Cloudly runs the schedule; an operator's stored value for the key wins over the template's. Cloudly 33.21.4 derives the owner of a launcher-environment secret file from the identity the workload runs as (AGL-459): every launcher delivery was published 0:0 and 0400, so a workload that runs as another user could not read its secrets and WorkloadInit exited with SECRET_SOURCE. The manifest publication now derives the owner per target cluster from the spec's runAs and the pinned image's config.User with @serve.zone/interfaces 32.60.0 deriveLauncherSecretDeliveryOwner, an owner that does not derive is refused launcher-secret-ownership-unknown, and the manifests 33.21.3 published are published again on upgrade. On @serve.zone/interfaces 32.60.0, external registry answers no longer carry the stored credential, Test connection proves the credentials through the Docker Registry v2 login, and a service update merges only the fields it states, so Edit and Configure Mail work again for a service with a runtime spec or an immutable image. Cloudly 33.21.4 also fixes the dashboard (AGL-387 to AGL-401: the status bar, the fonts, the dropdown presets, the view actions, the Secrets wizard and index, the Logs view and the App Store view), refuses a deployment without an existing service and node, a node action the node can never receive and a Jump code for a cluster without a relay, and bootstraps a node with the pinned Spark 35.1.18 release. Cloudly 33.21.4 adds no data step and keeps the data version 0.32.0, so an upgrade from 1.29.7 runs no data step; it adds no index, no collection and no setting, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is that of 1.29.7 with CLOUDLY_BACKUP_CRON and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and an upgrade from 1.29.6 or earlier runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0, which has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. 1.29.8 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.8 still exports the ledger with its documents; 1.30.4 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.8 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.4, and 1.29.8 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2 or 1.30.3, which sort above it. Like 1.29.7, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.7 the upgrade moves Cloudly from 33.21.3 to 33.21.4 and turns the scheduled workload backups on. From 1.29.6 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.7, the Cloudly 33.21.3 template, and describes the changes of Cloudly 33.21.3 and the releases before it that an upgrade from 1.29.6 or earlier also takes: Move Cloudly to 33.21.3, a patch over 33.21.2 that adds one forward-only data step, on the config of 1.29.6: 1.29.7 differs from 1.29.6 only in its image and changelog. A service migrated from Swarm reaches its object storage at its node's Corestore endpoint again. The Secrets v2 cutover turned the Docker-era Corestore binding environment, the nine object-storage endpoint names and, for a binding with retention, the eleven S3_RETENTION_* names, into service-owned secrets in scope platform:cloudly-corestore, the endpoint names with the Docker host corestore; a Corestore binding now gives the endpoint names as plain environment composed from the node's endpoint, and the secrets the cutover left behind replaced that environment in the workload, since a secret wins over an inherited variable of the same name, so such a service exited at startup with getaddrinfo ENOTFOUND corestore. Cloudly 33.21.3 adds the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. It retires every active service-owned secret in that scope whose key is one of the twenty non-secret object-storage names (the nine endpoint names and the eleven S3_RETENTION_* names) as a lifecycle change retires one: the version is retired and kept, never purged, and the metadata names the step as the author. The service's secrets revision advances once, and its desired manifests are published again without those secrets under a new secret rollout, the previous one accepted for five more minutes, so the runtime replaces the service's replicas once; a single-replica service restarts briefly. A service whose image plan or secret deployment plan the runtime already refuses gets its manifest from its next promotion. The six credential names, other scopes, other owners and greenfield services are left alone; a database binding gives no plain name, so no database secret is touched. Each service is one transaction, so a rerun writes nothing. The step has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. Cloudly 33.21.3 adds no index, no collection, no setting and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.6 and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and every upgrade to this template runs the step to 0.32.0. 1.29.7 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.7 still exports the ledger with its documents; 1.30.3 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.7 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.3, and 1.29.7 is never the upgrade target of 1.30.0, 1.30.1 or 1.30.2, which sort above it. Like 1.29.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.6 the upgrade moves Cloudly from 33.21.2 to 33.21.3 and runs the step to 0.32.0. From 1.29.5 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.6, the Cloudly 33.21.2 template, and describes the changes of Cloudly 33.21.2 and the releases before it that an upgrade from 1.29.5 or earlier also takes: Move Cloudly to 33.21.2, a fixes-only patch over 33.21.1, on the config of 1.29.5: 1.29.6 differs from 1.29.5 only in its image and changelog. A recovery redeploy over a dead retained rollout plan, one that no runtime can execute (such as a Swarm-era plan) and whose operation is settled, is no longer refused PLATFORM_CAPABILITY_UNAVAILABLE by its reserve and promote preflights while its Corestore bindings wait for their credentials: such a service has no plan a runtime can run, so it is issued no credential grant until a promotion writes one, and its database and object-storage bindings could not become ready before the promotion those preflights refused. The reserve and promote preflights now defer the readiness of such a binding while it is requested or provisioning, as the promote preflight of a held service's first promotion already did, and warn PLATFORM_BINDING_READINESS_DEFERRED; a binding that adopts a legacy resource is deferred only while Corestore lists that resource as authorized for it, and the route preflight after the rollout still requires every binding ready. The retained plan is judged by the operation it names, as the promotion fence reads it, never by the service's latest operation. 1.29.6 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.6 still exports the ledger with its documents; 1.30.2 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.6 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.2, and 1.29.6 is never the upgrade target of 1.30.0 or 1.30.1, which sort above it. Cloudly 33.21.2 adds no data step, no index and no setting and keeps the data version 0.31.0, so an upgrade from 1.29.5, 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.5 and requires Onebox 33.8.0. Like 1.29.5, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.5 the upgrade moves Cloudly from 33.21.1 to 33.21.2. From 1.29.4 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5, the Cloudly 33.21.1 template, and describes the changes of Cloudly 33.21.1 and the releases before it that an upgrade from 1.29.4 or earlier also takes: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
+              "changelog": "Move Cloudly to 33.26.0 on 1.30.7's configuration: 1.30.8 differs from 1.30.7 only in its image and changelog. The image is the published, independently qualified multi-platform index b5b1d9f0ba494d4e8472f141c03b7b76a8590d256debfd9c144f1eb866c18069, not a rebuild or a floating tag. Cloudly 33.26.0 adds the independently owned seven-method native Socket9 monitor at exact /typedrequest/native/v9, alongside the unchanged legacy Socket8 surfaces. It reuses the current password-login and inventory/log-read callbacks and their authority: a full native monitor requires a verified administrator; its identity is not a newly scoped read-only account. Native admission requires exact Host and Origin when present, denies client tags and unknown methods, waits for lifecycle readiness and drains admitted callbacks before dependencies stop. The release retains interfaces 32.65 and data version 0.37.0, updates the compatible rendering/validation security closure and qualifies the immutable 8.0.2 Node24 runtime, native dependencies and packaged startup on both architectures. Image scans retained three medium findings per architecture with no critical or high findings; publication is not live hosting or reviewer-access qualification. The predecessor image in 1.30.7 is Cloudly 33.24.0/data 0.33.0. An upgrade runs four forward-only steps: service-hostname-canonical (0.33.0 to 0.34.0), runtime-authority-tokens (0.34.0 to 0.35.0), runtime-assignment-failures (0.35.0 to 0.36.0), and bounded-slot-history (0.36.0 to 0.37.0). They canonicalize persisted service hostnames, create runtime authority-token records, capture retained assignments' accepted failures and prune slot attempt history to the incumbent plus eight ancestors while releasing the pruned attempts' logs. The Cloudly 33.25.1 fixes included here also serialize task runs, preserve cancelled task state, renew stable ingress certificates, bound node deliveries, fence cluster/node changes, serialize CoreMail publication, resume interrupted cleanup and avoid read-side lifecycle writes. Stop Cloudly and take a verified pre-upgrade database backup before any existing-instance upgrade. These data steps have no inverse and earlier images refuse the newer data version: return to 1.30.7 or any earlier template by restoring that pre-upgrade backup, never by selecting an older image over the migrated database. Keep breaking, requiresManualReview, migrationRequired and backupBeforeUpgrade set. The environment, storage mapping, secret-file delivery, WorkloadInit approval, platform OIDC, health check, Onebox minimum 33.8.0 and required storage feature ids are unchanged from 1.30.7. A host must fulfil storage.database.backup-exclusions and storage.object-storage.backup-exclusions; one without the database feature keeps 1.29.11 as its newest installable Cloudly target and receives no new template for the older feature set in this release. CLOUDLY_BACKUP_CRON stays 15 2 * * * for workload database backups (not their object storage). Workload objects below runtime-logs/ and the six runtime log ledger collections stay excluded from backups. Fresh installations use their own empty NoSQLDB namespace, bucket, keyring and hosted initial-admin flow; no production state or credentials are copied. Preserve both externally supplied encrypted root-readable Secrets v2 keyring and approved WorkloadInit files. A catalog publication does not install this version, upgrade a production control plane or authorize a public route. For earlier upgrades retain 1.30.7's documented backup/restore and peer-order requirements; cloudly@1.28.2 holds the older cumulative history.",
               "breaking": true,
               "requiresManualReview": true,
               "migrationRequired": true,
@@ -11657,12 +12493,12 @@
                 "expectedStatus": 200
               }
             },
-            "configDigest": "sha256:0c8c9268d7556b65c0340c3c2f8b9813001bdd86441c01708531fb46ee4a46e3"
+            "configDigest": "sha256:7db9e9b24d3d16c16502d24f1e72c48d451f81aef8c91a45591012a195487c24"
           },
           {
-            "version": "1.30.7",
+            "version": "1.30.9",
             "config": {
-              "image": "code.foss.global/serve.zone/cloudly@sha256:e230729d762f770d20643e05d769d92ad27038dc64cb9fac448f45b2f881b43a",
+              "image": "code.foss.global/serve.zone/cloudly@sha256:b5b1d9f0ba494d4e8472f141c03b7b76a8590d256debfd9c144f1eb866c18069",
               "targetPorts": [
                 {
                   "name": "web",
@@ -11799,6 +12635,19 @@
                   "value": "15 2 * * *",
                   "description": "Schedule of Cloudly's backup-all-services task, which backs up every workload service's databases (not its object storage); Cloudly registers the task only while this is set. Daily at 02:15 UTC, clear of Onebox's backup-archive-prune at 03:00.",
                   "required": false
+                },
+                {
+                  "key": "SERVEZONE_ADMINACCOUNT",
+                  "description": "Optional private first-start Cloudly operator in username:password form. Required by the fresh empty-instance workflow, never the reviewer credential. Supply only through sealed secret input; no default or automatic generation. Remove the bootstrap input with the Onebox owner command after first boot; revoking its persisted Cloudly password is a separate fresh-human credential action. Existing installations leave this unset.",
+                  "required": false,
+                  "secret": true,
+                  "delivery": {
+                    "type": "launcher-environment",
+                    "variableName": "SERVEZONE_ADMINACCOUNT",
+                    "uid": 0,
+                    "gid": 0,
+                    "mode": 256
+                  }
                 }
               ],
               "platformRequirements": {
@@ -11852,7 +12701,7 @@
                 "clientAuthenticationMethod": "client_secret_basic"
               },
               "minOneboxVersion": "33.8.0",
-              "changelog": "Declare Cloudly's runtime log ledger non-backup data without moving the image: this template runs the Cloudly 33.24.0 image of 1.29.11 and differs from 1.29.11 only in its changelog and two members. databaseBackupExclusions names the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. Every Cloudly start writes the log record of each running attempt that has none before it serves node requests, so the nodes' next batches are kept again after a restore, and the first batch of each capture states the numbers the restore lost as a gap. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.11; a host without the object-storage feature keeps 1.28.4. Like 1.29.11, this template sets CLOUDLY_BACKUP_CRON to 15 2 * * *, so Cloudly backs up the databases of its workload services daily at 02:15 UTC (not their object storage, AGL-473). From 1.29.11 the upgrade moves the template only: Cloudly runs the same image on the same data and no data step runs. From 1.30.6 it moves Cloudly from 33.23.2 to 33.24.0, keeps the declaration and runs no data step. From 1.30.5 it moves Cloudly from 33.23.0 to 33.24.0, keeps the declaration and runs no data step. From 1.30.4 it moves Cloudly from 33.21.4 to 33.24.0, keeps the declaration and runs the forward-only data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0. From 1.30.3 it moves Cloudly from 33.21.3 to 33.24.0, keeps the declaration, turns the scheduled workload backups on and runs that step. From 1.30.2 it moves Cloudly from 33.21.2 to 33.24.0, keeps the declaration, turns them on and runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0 and then that step. From 1.30.1 it moves Cloudly from 33.21.1 to 33.24.0 and from 1.30.0 from 33.21.0, keeps the declaration, turns them on and runs both steps. Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. From 1.29.10 or earlier the template also gains the declaration and the upgrade takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.11 and holds for this template: Move Cloudly to 33.24.0 on the config of 1.29.10: 1.29.11 differs from 1.29.10 only in its image and changelog. Cloudly 33.24.0 adds Temp tokens to the dashboard: Security, Temp tokens (/temp-tokens) mints, lists and revokes delegated admin temp tokens, so an administrator who signs in with a passkey through the platform provider (Onebox) mints one without a Cloudly password. A mint states its scopes (Secrets, Service credentials), its secret key prefixes (only with Secrets, at most 16) and a lifetime of 1 to 60 minutes. The token is shown once, in the answer's dialog, which drops it when it closes; the dashboard never writes it to a state part, browser storage, the URL or a log. Minting still needs a sign-in within the last five minutes, the rule Cloudly already applied and keeps unchanged: the page says until when the session can mint and offers Sign in again outside that window, and a provider session older than five minutes is signed into again at the provider first. The list shows the active tokens newest first, and Revoke asks for confirmation first. The release adds no data step, no environment variable, no setting, no collection and no index and changes neither the Dockerfile nor the qenv.yml requirements, so 1.29.11 keeps 1.29.10's runtime contract, CLOUDLY_BACKUP_CRON 15 2 * * * included, the data version 0.33.0 and the Onebox minimum 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and an upgrade from 1.29.8 or earlier runs the forward-only data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0, which has no way back: Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.29.8, 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. 1.29.11 does not declare the runtime log ledger databaseBackupExclusions; 1.30.7 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.11 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.7, and 1.29.11 is never the upgrade target of 1.30.0 to 1.30.6, which sort above it. Like 1.29.10, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.10 the upgrade moves Cloudly from 33.23.2 to 33.24.0 and runs no data step. From 1.29.9 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.10, the Cloudly 33.23.2 template, and describes the changes of Cloudly 33.23.2 and the releases before it that an upgrade from 1.29.9 or earlier also takes: Move Cloudly to 33.23.2, a fixes-only patch over 33.23.0, on the config of 1.29.9: 1.29.10 differs from 1.29.9 only in its image and changelog. Cloudly 33.23.2 takes the changes of Cloudly 33.23.1, which no template ran. Cloudly 33.23.1 composes each service's status on its own: a service whose stored image deployment plan, rollout status or newest deployment operation violates the @serve.zone/interfaces contract in a field the status reads gets the status failed with one composition stage that names the violation, for example: Cloudly cannot compose this service's status: its stored image deployment plan violates the contract (releaseTag, operationId missing). A composition that throws gives that service the same failed status with the reason and one error log line naming the service, and every other service keeps its status. Cloudly 33.23.0 answered the administrator's services list, getServiceById of such a service and the MCP tool cloudly_list_services with an internal error when one service stored a plan written before the plan carried releaseTag and operationId. The stored documents are not changed; a new promotion replaces such a plan. Cloudly 33.23.2 lets servezone service credential ensure give a service whose SERVEZONE_PLATFORM_AUTHORIZATION an operator stored as an ordinary secret its service machine credential: ensure, run by a human administrator who signed in within five minutes or with a serviceCredentials temp token, takes that operator-managed row over in the same transaction, active, retired, revoked or deleted, revokes every legacy version that could still be issued, keeps every version, moves the row to the credential's platform scope stamped with the administrator and the token, records the takeover on the new version, publishes the service's secret manifests again and is fenced on the operator authority and revision it read. Before, ensure was refused SERVICE_CREDENTIAL_CONFLICT for such a service whatever an administrator did first. A secret of the key under any other authority is still refused, and the legacy user token itself is not revoked automatically. Neither release adds a data step, an environment variable, a setting, a collection or an index, and neither changes the Dockerfile or the qenv.yml requirements, so 1.29.10 keeps 1.29.9's runtime contract, CLOUDLY_BACKUP_CRON 15 2 * * * included, the data version 0.33.0 and the Onebox minimum 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and an upgrade from 1.29.8 or earlier runs the forward-only data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0, which has no way back: Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.29.8, 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. 1.29.10 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.10 still exports the ledger with its documents; 1.30.6 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.10 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.6, and 1.29.10 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2, 1.30.3, 1.30.4 or 1.30.5, which sort above it. Like 1.29.9, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.9 the upgrade moves Cloudly from 33.23.0 to 33.23.2 and runs no data step. From 1.29.8 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.9, the Cloudly 33.23.0 template, and describes the changes of Cloudly 33.23.0 and the releases before it that an upgrade from 1.29.8 or earlier also takes: Move Cloudly to 33.23.0 on the config of 1.29.8: 1.29.9 differs from 1.29.8 only in its image and changelog. Cloudly 33.23.0 adds one forward-only data step, retire-stale-reconciliation-stages from 0.32.0 to 0.33.0, and takes the changes of Cloudly 33.22.0, which no template ran. A service's status is now what the producers Cloudly runs state about it: the Pallet runtime's verdict on each target, the current plan's rollout, the newest deployment operation and the records of the observers Cloudly runs (externalGateway, externalGatewayMail, coreMail and platformBinding). The Services list, the Overview and the MCP tool cloudly_list_services read that one status, so a service that showed Failed or Converging for good because of a stage the Swarm-era Coreflow reconciler last reported on 2026-08-03 shows its real state. getServices composes the status for an administrator only; a cluster's relay gets the services without it. The Overview lists every failed service with its reason and a link to it (degraded ones as warnings); its Deployments tile and Recent deployment changes table are gone, and so is the dashboard summary's deployments count. Cloudly no longer serves reportReconciliationStatus and reportDeploymentSnapshots, the Swarm-era Coreflow intake no component sends, and an observer deletes its record when what it observes goes: a route withdrawn from a service and a deleted or tombstoned platform binding withdraw their externalGateway and platformBinding record. The data step deletes, through the database driver before any model is prepared, every reconciliation record of a kind only the Swarm-era Coreflow reconciler wrote (workload, routing, immutable-image, immutable-adoption and immutable-attestation) and every externalGateway record of a route its service no longer declares; a record of a route still declared stays, failed or not, so do the records of other observers, and a rerun removes nothing. The deployment operations get the index service_deployment_operation_newest, which the model builds at start. Cloudly 33.23.0 adds delegated admin temp tokens: a human administrator who signed in within five minutes mints a token for the scopes secrets and serviceCredentials that lives one hour at most (servezone admin temp-token mint, list and revoke); Cloudly stores only its SHA-256, in cloudly_delegated_admin_tokens, records every use, refusal and failed write in cloudly_delegated_admin_token_audit, and a token never writes a reserved name and never deletes. Cloudly 33.23.0 moves to @serve.zone/interfaces 32.61.0 and its dashboard to @design.estate/dees-catalog 23 (controls are 30px tall instead of 32px, dialogs and sheets have 14px corners instead of 20px). With Cloudly 33.22.0 the scheduled backup-all-services task, which 1.29.8 turned on with CLOUDLY_BACKUP_CRON, backs up each workload service on the node and cluster its data lives on: on a Cloudly with more than one cluster every service was refused, and on one with a single cluster every backup was refused backup-node-required, while the run was recorded completed. A service with nothing to back up is skipped, a service without a target fails by name, and a run in which any service failed ends failed and names each one. Cloudly 33.22.0 adds servezone service credential inspect, ensure, rotate and revoke for the service machine credential @serve.zone/platformclient 4.0.0 and later signs in with, and names a refused actor SERVICE_CREDENTIAL_UNAUTHORIZED. Neither release adds an environment variable or a setting, and neither changes the Dockerfile or the qenv.yml requirements, so 1.29.9 keeps 1.29.8's runtime contract, CLOUDLY_BACKUP_CRON 15 2 * * * included, and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: the data step has no way back. Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.29.8, 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. 1.29.9 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.9 still exports the ledger with its documents; 1.30.5 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.9 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.5, and 1.29.9 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2, 1.30.3 or 1.30.4, which sort above it. Like 1.29.8, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.8 the upgrade moves Cloudly from 33.21.4 to 33.23.0 and runs the data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0. From 1.29.7 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.8, the Cloudly 33.21.4 template, and describes the changes of Cloudly 33.21.4 and the releases before it that an upgrade from 1.29.7 or earlier also takes: Move Cloudly to 33.21.4, a fixes-only patch over 33.21.3, on the config of 1.29.7, and turn on Cloudly's scheduled workload backups: 1.29.8 differs from 1.29.7 only in its image, its changelog and the environment variable CLOUDLY_BACKUP_CRON. Cloudly registers its backup-all-services task only when CLOUDLY_BACKUP_CRON is set, and no earlier template set it, so the databases of the workload services Cloudly runs were never backed up on a schedule. This template sets it to 15 2 * * *: every day at 02:15 UTC, clear of Onebox's backup-archive-prune at 03:00, Cloudly creates a backup of every workload service with backup-enabled resources, and it keeps the newest 24 settled backups of each service (CLOUDLY_BACKUP_KEEP_LAST, which the template does not set, defaults to 24) and deletes older ones with their stored archives. These backups cover the workload databases; a workload's object storage is not part of them (AGL-473). The variable is a plain declaration with no secret and no delivery and is not required. A host that upgrades a service gives a key the service does not have yet the template's value, so an upgraded Cloudly runs the schedule; an operator's stored value for the key wins over the template's. Cloudly 33.21.4 derives the owner of a launcher-environment secret file from the identity the workload runs as (AGL-459): every launcher delivery was published 0:0 and 0400, so a workload that runs as another user could not read its secrets and WorkloadInit exited with SECRET_SOURCE. The manifest publication now derives the owner per target cluster from the spec's runAs and the pinned image's config.User with @serve.zone/interfaces 32.60.0 deriveLauncherSecretDeliveryOwner, an owner that does not derive is refused launcher-secret-ownership-unknown, and the manifests 33.21.3 published are published again on upgrade. On @serve.zone/interfaces 32.60.0, external registry answers no longer carry the stored credential, Test connection proves the credentials through the Docker Registry v2 login, and a service update merges only the fields it states, so Edit and Configure Mail work again for a service with a runtime spec or an immutable image. Cloudly 33.21.4 also fixes the dashboard (AGL-387 to AGL-401: the status bar, the fonts, the dropdown presets, the view actions, the Secrets wizard and index, the Logs view and the App Store view), refuses a deployment without an existing service and node, a node action the node can never receive and a Jump code for a cluster without a relay, and bootstraps a node with the pinned Spark 35.1.18 release. Cloudly 33.21.4 adds no data step and keeps the data version 0.32.0, so an upgrade from 1.29.7 runs no data step; it adds no index, no collection and no setting, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is that of 1.29.7 with CLOUDLY_BACKUP_CRON and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and an upgrade from 1.29.6 or earlier runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0, which has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. 1.29.8 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.8 still exports the ledger with its documents; 1.30.4 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.8 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.4, and 1.29.8 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2 or 1.30.3, which sort above it. Like 1.29.7, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.7 the upgrade moves Cloudly from 33.21.3 to 33.21.4 and turns the scheduled workload backups on. From 1.29.6 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.7, the Cloudly 33.21.3 template, and describes the changes of Cloudly 33.21.3 and the releases before it that an upgrade from 1.29.6 or earlier also takes: Move Cloudly to 33.21.3, a patch over 33.21.2 that adds one forward-only data step, on the config of 1.29.6: 1.29.7 differs from 1.29.6 only in its image and changelog. A service migrated from Swarm reaches its object storage at its node's Corestore endpoint again. The Secrets v2 cutover turned the Docker-era Corestore binding environment, the nine object-storage endpoint names and, for a binding with retention, the eleven S3_RETENTION_* names, into service-owned secrets in scope platform:cloudly-corestore, the endpoint names with the Docker host corestore; a Corestore binding now gives the endpoint names as plain environment composed from the node's endpoint, and the secrets the cutover left behind replaced that environment in the workload, since a secret wins over an inherited variable of the same name, so such a service exited at startup with getaddrinfo ENOTFOUND corestore. Cloudly 33.21.3 adds the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. It retires every active service-owned secret in that scope whose key is one of the twenty non-secret object-storage names (the nine endpoint names and the eleven S3_RETENTION_* names) as a lifecycle change retires one: the version is retired and kept, never purged, and the metadata names the step as the author. The service's secrets revision advances once, and its desired manifests are published again without those secrets under a new secret rollout, the previous one accepted for five more minutes, so the runtime replaces the service's replicas once; a single-replica service restarts briefly. A service whose image plan or secret deployment plan the runtime already refuses gets its manifest from its next promotion. The six credential names, other scopes, other owners and greenfield services are left alone; a database binding gives no plain name, so no database secret is touched. Each service is one transaction, so a rerun writes nothing. The step has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. Cloudly 33.21.3 adds no index, no collection, no setting and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.6 and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and every upgrade to this template runs the step to 0.32.0. 1.29.7 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.7 still exports the ledger with its documents; 1.30.3 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.7 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.3, and 1.29.7 is never the upgrade target of 1.30.0, 1.30.1 or 1.30.2, which sort above it. Like 1.29.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.6 the upgrade moves Cloudly from 33.21.2 to 33.21.3 and runs the step to 0.32.0. From 1.29.5 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.6, the Cloudly 33.21.2 template, and describes the changes of Cloudly 33.21.2 and the releases before it that an upgrade from 1.29.5 or earlier also takes: Move Cloudly to 33.21.2, a fixes-only patch over 33.21.1, on the config of 1.29.5: 1.29.6 differs from 1.29.5 only in its image and changelog. A recovery redeploy over a dead retained rollout plan, one that no runtime can execute (such as a Swarm-era plan) and whose operation is settled, is no longer refused PLATFORM_CAPABILITY_UNAVAILABLE by its reserve and promote preflights while its Corestore bindings wait for their credentials: such a service has no plan a runtime can run, so it is issued no credential grant until a promotion writes one, and its database and object-storage bindings could not become ready before the promotion those preflights refused. The reserve and promote preflights now defer the readiness of such a binding while it is requested or provisioning, as the promote preflight of a held service's first promotion already did, and warn PLATFORM_BINDING_READINESS_DEFERRED; a binding that adopts a legacy resource is deferred only while Corestore lists that resource as authorized for it, and the route preflight after the rollout still requires every binding ready. The retained plan is judged by the operation it names, as the promotion fence reads it, never by the service's latest operation. 1.29.6 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.6 still exports the ledger with its documents; 1.30.2 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.6 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.2, and 1.29.6 is never the upgrade target of 1.30.0 or 1.30.1, which sort above it. Cloudly 33.21.2 adds no data step, no index and no setting and keeps the data version 0.31.0, so an upgrade from 1.29.5, 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.5 and requires Onebox 33.8.0. Like 1.29.5, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.5 the upgrade moves Cloudly from 33.21.1 to 33.21.2. From 1.29.4 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5, the Cloudly 33.21.1 template, and describes the changes of Cloudly 33.21.1 and the releases before it that an upgrade from 1.29.4 or earlier also takes: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
+              "changelog": "Adds one optional, sealed owner-supplied private first-start operator input through the existing SERVEZONE_ADMINACCOUNT path. This is interim compatibility for the exact Cloudly33.26.0 image only, the transitional delivery alternative recorded in AGL-1246. AGL-1179 under AGL-1178 owns the final Onebox sign-in broker and removes both Cloudly password-admin and parent initial-admin bootstrap paths; this declaration does not implement or extend that final architecture. This is a declaration-only successor on the exact same published Cloudly33.26.0 image. It adds no parent-JWT delivery, launcher, public environment default or generated credential. Fresh empty installs require the owner input for first boot; existing installations and upgrades omit it. After first boot use the existing Onebox service patch-env owner command to remove this encrypted startup alias and replace the runtime; persisted application passwords are revoked only by an explicit operator action through Cloudly fresh-human credential management after real hosted OIDC access is established, never automatically. This template does not qualify a fresh host or AGL-1246's separate install/upgrade lab and catalog targets on Cloudly>=33.27.0.\n\nThe rest of this changelog is that of 1.30.8 and holds for this template: Move Cloudly to 33.26.0 on 1.30.7's configuration: 1.30.8 differs from 1.30.7 only in its image and changelog. The image is the published, independently qualified multi-platform index b5b1d9f0ba494d4e8472f141c03b7b76a8590d256debfd9c144f1eb866c18069, not a rebuild or a floating tag. Cloudly 33.26.0 adds the independently owned seven-method native Socket9 monitor at exact /typedrequest/native/v9, alongside the unchanged legacy Socket8 surfaces. It reuses the current password-login and inventory/log-read callbacks and their authority: a full native monitor requires a verified administrator; its identity is not a newly scoped read-only account. Native admission requires exact Host and Origin when present, denies client tags and unknown methods, waits for lifecycle readiness and drains admitted callbacks before dependencies stop. The release retains interfaces 32.65 and data version 0.37.0, updates the compatible rendering/validation security closure and qualifies the immutable 8.0.2 Node24 runtime, native dependencies and packaged startup on both architectures. Image scans retained three medium findings per architecture with no critical or high findings; publication is not live hosting or reviewer-access qualification. The predecessor image in 1.30.7 is Cloudly 33.24.0/data 0.33.0. An upgrade runs four forward-only steps: service-hostname-canonical (0.33.0 to 0.34.0), runtime-authority-tokens (0.34.0 to 0.35.0), runtime-assignment-failures (0.35.0 to 0.36.0), and bounded-slot-history (0.36.0 to 0.37.0). They canonicalize persisted service hostnames, create runtime authority-token records, capture retained assignments' accepted failures and prune slot attempt history to the incumbent plus eight ancestors while releasing the pruned attempts' logs. The Cloudly 33.25.1 fixes included here also serialize task runs, preserve cancelled task state, renew stable ingress certificates, bound node deliveries, fence cluster/node changes, serialize CoreMail publication, resume interrupted cleanup and avoid read-side lifecycle writes. Stop Cloudly and take a verified pre-upgrade database backup before any existing-instance upgrade. These data steps have no inverse and earlier images refuse the newer data version: return to 1.30.7 or any earlier template by restoring that pre-upgrade backup, never by selecting an older image over the migrated database. Keep breaking, requiresManualReview, migrationRequired and backupBeforeUpgrade set. The environment, storage mapping, secret-file delivery, WorkloadInit approval, platform OIDC, health check, Onebox minimum 33.8.0 and required storage feature ids are unchanged from 1.30.7. A host must fulfil storage.database.backup-exclusions and storage.object-storage.backup-exclusions; one without the database feature keeps 1.29.11 as its newest installable Cloudly target and receives no new template for the older feature set in this release. CLOUDLY_BACKUP_CRON stays 15 2 * * * for workload database backups (not their object storage). Workload objects below runtime-logs/ and the six runtime log ledger collections stay excluded from backups. Fresh installations use their own empty NoSQLDB namespace, bucket, keyring and hosted initial-admin flow; no production state or credentials are copied. Preserve both externally supplied encrypted root-readable Secrets v2 keyring and approved WorkloadInit files. A catalog publication does not install this version, upgrade a production control plane or authorize a public route. For earlier upgrades retain 1.30.7's documented backup/restore and peer-order requirements; cloudly@1.28.2 holds the older cumulative history.",
               "breaking": true,
               "requiresManualReview": true,
               "migrationRequired": true,
@@ -11867,7 +12716,7 @@
                 "expectedStatus": 200
               }
             },
-            "configDigest": "sha256:cf68bdbc782b8739aa90067b048bea1cf2890b737d0cbb7f008414995b136cef"
+            "configDigest": "sha256:7d88ac5ff1084b91a0b52799e681d29d1c852fcaefe4ad03d0f346e217f2a1f6"
           }
         ]
       },
@@ -12641,11 +13490,11 @@
       }
     ]
   },
-  "catalogDigest": "sha256:bab82e8710c9d5b4f681e1b26cef187f875f80b9763a90481bf65c36a4de5fc7",
+  "catalogDigest": "sha256:734e72bed1384421d8a30821ffce022aa34715fec1e0bd5d7e8a71897bf73b29",
   "signatures": [
     {
       "keyId": "sha256:58956c3537cdf542430d6685261824776d6128ebb9a93b3d77b53f2d031ba854",
-      "signature": "a854177de9ed69502fe61f05146b9662429e95fbc480e5710e3f8a6428a6c25a337190f53894abfa745a6fc0975ec95dfce114d495c2f8fc123b69000e3df00a"
+      "signature": "499cd9b917a42502544a324acb4c96e97347714af12f71ac7e6e13c3cde5cdf9b9b6c596ac5176eecf6111d13959b3f0b32bb46191e6929303f989f1e8d61f0e"
     }
   ]
 }