@serve.zone/appstore 33.15.1 → 33.17.0

package/dist_catalog/catalog.json

@@ -4,9 +4,9 @@
     "major": 1,
     "minor": 0
   },
-  "revision": 33000015000001,
-  "publishedAt": "2026-10-20T17:07:47.000Z",
-  "sourceRevision": "d8721b78424db170ee7decb54bd7d4b1a5559796",
+  "revision": 33000017000000,
+  "publishedAt": "2026-10-21T19:07:12.000Z",
+  "sourceRevision": "fb197beffcd1527bebb85757d916fbd53b26777f",
   "apps": [
     {
       "info": {
@@ -59,7 +59,7 @@
           "Docs": "https://serve.zone"
         }
       },
-      "latestVersion": "1.30.3",
+      "latestVersion": "1.30.5",
       "versions": [
         {
           "version": "1.0.0",
@@ -9432,10 +9432,604 @@
           },
           "configDigest": "sha256:385e93003bc7b064ed83fb725db89ec881a03fc424e3415801b0a47f8c40c3ba"
         },
+        {
+          "version": "1.29.8",
+          "config": {
+            "image": "code.foss.global/serve.zone/cloudly@sha256:e8e0f2b5a9e3d7aff4ba3201e358e99900b5c3792c24873b3dc0b3cb3f921439",
+            "targetPorts": [
+              {
+                "name": "web",
+                "port": 3000,
+                "protocol": "http",
+                "default": true
+              }
+            ],
+            "envVars": [
+              {
+                "key": "SERVEZONE_ENVIRONMENT",
+                "value": "production",
+                "description": "Cloudly runtime environment.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_PUBLIC_ORIGIN",
+                "value": "https://${SERVICE_DOMAIN}",
+                "description": "Canonical public Cloudly origin including protocol and optional public port.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_LISTENPORT",
+                "value": "3000",
+                "description": "Internal Cloudly HTTP port inside the container.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_SSLMODE",
+                "value": "external",
+                "description": "Use external TLS termination through Onebox or dcrouter.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_SECRET_KEYRING_PATH",
+                "value": "/run/secrets/cloudly-secrets-v2-keyring.json",
+                "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_SECRET_KEYRING_FILE",
+                "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.",
+                "required": true,
+                "secret": true,
+                "delivery": {
+                  "type": "file",
+                  "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json",
+                  "uid": 0,
+                  "gid": 0,
+                  "mode": 256
+                }
+              },
+              {
+                "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH",
+                "value": "/run/secrets/cloudly-workloadinit-approval.json",
+                "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE",
+                "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.",
+                "required": true,
+                "secret": true,
+                "delivery": {
+                  "type": "file",
+                  "targetPath": "/run/secrets/cloudly-workloadinit-approval.json",
+                  "uid": 0,
+                  "gid": 0,
+                  "mode": 256
+                }
+              },
+              {
+                "key": "MONGODB_URL",
+                "value": "${MONGODB_URI}",
+                "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "MONGODB_NAME",
+                "value": "${MONGODB_DATABASE}",
+                "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "MONGODB_USER",
+                "value": "${MONGODB_USERNAME}",
+                "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "MONGODB_PASS",
+                "value": "${MONGODB_PASSWORD}",
+                "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_ENDPOINT",
+                "value": "onebox-corestore",
+                "description": "S3 endpoint host provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_PORT",
+                "value": "9000",
+                "description": "S3 endpoint port provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_USESSL",
+                "value": "false",
+                "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.",
+                "required": true
+              },
+              {
+                "key": "S3_BUCKET",
+                "value": "${S3_BUCKET}",
+                "description": "Cloudly's own S3 bucket, provisioned by Onebox Corestore: registry storage and kept workload logs.",
+                "required": true
+              },
+              {
+                "key": "S3_ACCESSKEY",
+                "value": "${S3_ACCESS_KEY}",
+                "description": "S3 access key provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_SECRETKEY",
+                "value": "${S3_SECRET_KEY}",
+                "description": "S3 secret key provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "CLOUDLY_BACKUP_CRON",
+                "value": "15 2 * * *",
+                "description": "Schedule of Cloudly's backup-all-services task, which backs up every workload service's databases (not its object storage); Cloudly registers the task only while this is set. Daily at 02:15 UTC, clear of Onebox's backup-archive-prune at 03:00.",
+                "required": false
+              }
+            ],
+            "platformRequirements": {
+              "mongodb": true,
+              "s3": true
+            },
+            "backupExclusions": [
+              {
+                "storageRequest": "legacy-s3",
+                "keyPrefixes": [
+                  "runtime-logs/"
+                ],
+                "reason": "Workload logs; a restore brings back state, not past logs."
+              }
+            ],
+            "platformOidc": {
+              "redirectPath": "/auth/oidc/callback",
+              "roles": [
+                {
+                  "id": "admin",
+                  "label": "Administrator",
+                  "description": "Full Cloudly administration for this hosted instance."
+                },
+                {
+                  "id": "user",
+                  "label": "User",
+                  "description": "Standard authenticated Cloudly access for this hosted instance."
+                }
+              ],
+              "environmentVariables": {
+                "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER",
+                "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID",
+                "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET",
+                "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI",
+                "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE"
+              },
+              "clientAuthenticationMethod": "client_secret_basic"
+            },
+            "minOneboxVersion": "33.8.0",
+            "changelog": "Move Cloudly to 33.21.4, a fixes-only patch over 33.21.3, on the config of 1.29.7, and turn on Cloudly's scheduled workload backups: 1.29.8 differs from 1.29.7 only in its image, its changelog and the environment variable CLOUDLY_BACKUP_CRON. Cloudly registers its backup-all-services task only when CLOUDLY_BACKUP_CRON is set, and no earlier template set it, so the databases of the workload services Cloudly runs were never backed up on a schedule. This template sets it to 15 2 * * *: every day at 02:15 UTC, clear of Onebox's backup-archive-prune at 03:00, Cloudly creates a backup of every workload service with backup-enabled resources, and it keeps the newest 24 settled backups of each service (CLOUDLY_BACKUP_KEEP_LAST, which the template does not set, defaults to 24) and deletes older ones with their stored archives. These backups cover the workload databases; a workload's object storage is not part of them (AGL-473). The variable is a plain declaration with no secret and no delivery and is not required. A host that upgrades a service gives a key the service does not have yet the template's value, so an upgraded Cloudly runs the schedule; an operator's stored value for the key wins over the template's. Cloudly 33.21.4 derives the owner of a launcher-environment secret file from the identity the workload runs as (AGL-459): every launcher delivery was published 0:0 and 0400, so a workload that runs as another user could not read its secrets and WorkloadInit exited with SECRET_SOURCE. The manifest publication now derives the owner per target cluster from the spec's runAs and the pinned image's config.User with @serve.zone/interfaces 32.60.0 deriveLauncherSecretDeliveryOwner, an owner that does not derive is refused launcher-secret-ownership-unknown, and the manifests 33.21.3 published are published again on upgrade. On @serve.zone/interfaces 32.60.0, external registry answers no longer carry the stored credential, Test connection proves the credentials through the Docker Registry v2 login, and a service update merges only the fields it states, so Edit and Configure Mail work again for a service with a runtime spec or an immutable image. Cloudly 33.21.4 also fixes the dashboard (AGL-387 to AGL-401: the status bar, the fonts, the dropdown presets, the view actions, the Secrets wizard and index, the Logs view and the App Store view), refuses a deployment without an existing service and node, a node action the node can never receive and a Jump code for a cluster without a relay, and bootstraps a node with the pinned Spark 35.1.18 release. Cloudly 33.21.4 adds no data step and keeps the data version 0.32.0, so an upgrade from 1.29.7 runs no data step; it adds no index, no collection and no setting, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is that of 1.29.7 with CLOUDLY_BACKUP_CRON and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and an upgrade from 1.29.6 or earlier runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0, which has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. 1.29.8 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.8 still exports the ledger with its documents; 1.30.4 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.8 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.4, and 1.29.8 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2 or 1.30.3, which sort above it. Like 1.29.7, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.7 the upgrade moves Cloudly from 33.21.3 to 33.21.4 and turns the scheduled workload backups on. From 1.29.6 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.7, the Cloudly 33.21.3 template, and describes the changes of Cloudly 33.21.3 and the releases before it that an upgrade from 1.29.6 or earlier also takes: Move Cloudly to 33.21.3, a patch over 33.21.2 that adds one forward-only data step, on the config of 1.29.6: 1.29.7 differs from 1.29.6 only in its image and changelog. A service migrated from Swarm reaches its object storage at its node's Corestore endpoint again. The Secrets v2 cutover turned the Docker-era Corestore binding environment, the nine object-storage endpoint names and, for a binding with retention, the eleven S3_RETENTION_* names, into service-owned secrets in scope platform:cloudly-corestore, the endpoint names with the Docker host corestore; a Corestore binding now gives the endpoint names as plain environment composed from the node's endpoint, and the secrets the cutover left behind replaced that environment in the workload, since a secret wins over an inherited variable of the same name, so such a service exited at startup with getaddrinfo ENOTFOUND corestore. Cloudly 33.21.3 adds the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. It retires every active service-owned secret in that scope whose key is one of the twenty non-secret object-storage names (the nine endpoint names and the eleven S3_RETENTION_* names) as a lifecycle change retires one: the version is retired and kept, never purged, and the metadata names the step as the author. The service's secrets revision advances once, and its desired manifests are published again without those secrets under a new secret rollout, the previous one accepted for five more minutes, so the runtime replaces the service's replicas once; a single-replica service restarts briefly. A service whose image plan or secret deployment plan the runtime already refuses gets its manifest from its next promotion. The six credential names, other scopes, other owners and greenfield services are left alone; a database binding gives no plain name, so no database secret is touched. Each service is one transaction, so a rerun writes nothing. The step has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. Cloudly 33.21.3 adds no index, no collection, no setting and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.6 and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and every upgrade to this template runs the step to 0.32.0. 1.29.7 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.7 still exports the ledger with its documents; 1.30.3 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.7 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.3, and 1.29.7 is never the upgrade target of 1.30.0, 1.30.1 or 1.30.2, which sort above it. Like 1.29.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.6 the upgrade moves Cloudly from 33.21.2 to 33.21.3 and runs the step to 0.32.0. From 1.29.5 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.6, the Cloudly 33.21.2 template, and describes the changes of Cloudly 33.21.2 and the releases before it that an upgrade from 1.29.5 or earlier also takes: Move Cloudly to 33.21.2, a fixes-only patch over 33.21.1, on the config of 1.29.5: 1.29.6 differs from 1.29.5 only in its image and changelog. A recovery redeploy over a dead retained rollout plan, one that no runtime can execute (such as a Swarm-era plan) and whose operation is settled, is no longer refused PLATFORM_CAPABILITY_UNAVAILABLE by its reserve and promote preflights while its Corestore bindings wait for their credentials: such a service has no plan a runtime can run, so it is issued no credential grant until a promotion writes one, and its database and object-storage bindings could not become ready before the promotion those preflights refused. The reserve and promote preflights now defer the readiness of such a binding while it is requested or provisioning, as the promote preflight of a held service's first promotion already did, and warn PLATFORM_BINDING_READINESS_DEFERRED; a binding that adopts a legacy resource is deferred only while Corestore lists that resource as authorized for it, and the route preflight after the rollout still requires every binding ready. The retained plan is judged by the operation it names, as the promotion fence reads it, never by the service's latest operation. 1.29.6 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.6 still exports the ledger with its documents; 1.30.2 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.6 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.2, and 1.29.6 is never the upgrade target of 1.30.0 or 1.30.1, which sort above it. Cloudly 33.21.2 adds no data step, no index and no setting and keeps the data version 0.31.0, so an upgrade from 1.29.5, 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.5 and requires Onebox 33.8.0. Like 1.29.5, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.5 the upgrade moves Cloudly from 33.21.1 to 33.21.2. From 1.29.4 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5, the Cloudly 33.21.1 template, and describes the changes of Cloudly 33.21.1 and the releases before it that an upgrade from 1.29.4 or earlier also takes: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
+            "breaking": true,
+            "requiresManualReview": true,
+            "migrationRequired": true,
+            "backupBeforeUpgrade": true,
+            "requiresFeatures": [
+              "storage.object-storage.backup-exclusions"
+            ],
+            "healthCheck": {
+              "path": "/status",
+              "port": 3000,
+              "expectedStatus": 200
+            }
+          },
+          "configDigest": "sha256:75801c074b8c4b7b41ac2bf9733a2bcbcad9fcbba1a78be864dfb1dcd0000ad4"
+        },
+        {
+          "version": "1.29.9",
+          "config": {
+            "image": "code.foss.global/serve.zone/cloudly@sha256:73be2fb57ee6bf40f83569c677f381713a7ddcc38fedbb3f16a7d85fb1e6e33d",
+            "targetPorts": [
+              {
+                "name": "web",
+                "port": 3000,
+                "protocol": "http",
+                "default": true
+              }
+            ],
+            "envVars": [
+              {
+                "key": "SERVEZONE_ENVIRONMENT",
+                "value": "production",
+                "description": "Cloudly runtime environment.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_PUBLIC_ORIGIN",
+                "value": "https://${SERVICE_DOMAIN}",
+                "description": "Canonical public Cloudly origin including protocol and optional public port.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_LISTENPORT",
+                "value": "3000",
+                "description": "Internal Cloudly HTTP port inside the container.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_SSLMODE",
+                "value": "external",
+                "description": "Use external TLS termination through Onebox or dcrouter.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_SECRET_KEYRING_PATH",
+                "value": "/run/secrets/cloudly-secrets-v2-keyring.json",
+                "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_SECRET_KEYRING_FILE",
+                "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.",
+                "required": true,
+                "secret": true,
+                "delivery": {
+                  "type": "file",
+                  "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json",
+                  "uid": 0,
+                  "gid": 0,
+                  "mode": 256
+                }
+              },
+              {
+                "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH",
+                "value": "/run/secrets/cloudly-workloadinit-approval.json",
+                "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE",
+                "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.",
+                "required": true,
+                "secret": true,
+                "delivery": {
+                  "type": "file",
+                  "targetPath": "/run/secrets/cloudly-workloadinit-approval.json",
+                  "uid": 0,
+                  "gid": 0,
+                  "mode": 256
+                }
+              },
+              {
+                "key": "MONGODB_URL",
+                "value": "${MONGODB_URI}",
+                "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "MONGODB_NAME",
+                "value": "${MONGODB_DATABASE}",
+                "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "MONGODB_USER",
+                "value": "${MONGODB_USERNAME}",
+                "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "MONGODB_PASS",
+                "value": "${MONGODB_PASSWORD}",
+                "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_ENDPOINT",
+                "value": "onebox-corestore",
+                "description": "S3 endpoint host provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_PORT",
+                "value": "9000",
+                "description": "S3 endpoint port provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_USESSL",
+                "value": "false",
+                "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.",
+                "required": true
+              },
+              {
+                "key": "S3_BUCKET",
+                "value": "${S3_BUCKET}",
+                "description": "Cloudly's own S3 bucket, provisioned by Onebox Corestore: registry storage and kept workload logs.",
+                "required": true
+              },
+              {
+                "key": "S3_ACCESSKEY",
+                "value": "${S3_ACCESS_KEY}",
+                "description": "S3 access key provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_SECRETKEY",
+                "value": "${S3_SECRET_KEY}",
+                "description": "S3 secret key provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "CLOUDLY_BACKUP_CRON",
+                "value": "15 2 * * *",
+                "description": "Schedule of Cloudly's backup-all-services task, which backs up every workload service's databases (not its object storage); Cloudly registers the task only while this is set. Daily at 02:15 UTC, clear of Onebox's backup-archive-prune at 03:00.",
+                "required": false
+              }
+            ],
+            "platformRequirements": {
+              "mongodb": true,
+              "s3": true
+            },
+            "backupExclusions": [
+              {
+                "storageRequest": "legacy-s3",
+                "keyPrefixes": [
+                  "runtime-logs/"
+                ],
+                "reason": "Workload logs; a restore brings back state, not past logs."
+              }
+            ],
+            "platformOidc": {
+              "redirectPath": "/auth/oidc/callback",
+              "roles": [
+                {
+                  "id": "admin",
+                  "label": "Administrator",
+                  "description": "Full Cloudly administration for this hosted instance."
+                },
+                {
+                  "id": "user",
+                  "label": "User",
+                  "description": "Standard authenticated Cloudly access for this hosted instance."
+                }
+              ],
+              "environmentVariables": {
+                "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER",
+                "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID",
+                "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET",
+                "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI",
+                "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE"
+              },
+              "clientAuthenticationMethod": "client_secret_basic"
+            },
+            "minOneboxVersion": "33.8.0",
+            "changelog": "Move Cloudly to 33.23.0 on the config of 1.29.8: 1.29.9 differs from 1.29.8 only in its image and changelog. Cloudly 33.23.0 adds one forward-only data step, retire-stale-reconciliation-stages from 0.32.0 to 0.33.0, and takes the changes of Cloudly 33.22.0, which no template ran. A service's status is now what the producers Cloudly runs state about it: the Pallet runtime's verdict on each target, the current plan's rollout, the newest deployment operation and the records of the observers Cloudly runs (externalGateway, externalGatewayMail, coreMail and platformBinding). The Services list, the Overview and the MCP tool cloudly_list_services read that one status, so a service that showed Failed or Converging for good because of a stage the Swarm-era Coreflow reconciler last reported on 2026-08-03 shows its real state. getServices composes the status for an administrator only; a cluster's relay gets the services without it. The Overview lists every failed service with its reason and a link to it (degraded ones as warnings); its Deployments tile and Recent deployment changes table are gone, and so is the dashboard summary's deployments count. Cloudly no longer serves reportReconciliationStatus and reportDeploymentSnapshots, the Swarm-era Coreflow intake no component sends, and an observer deletes its record when what it observes goes: a route withdrawn from a service and a deleted or tombstoned platform binding withdraw their externalGateway and platformBinding record. The data step deletes, through the database driver before any model is prepared, every reconciliation record of a kind only the Swarm-era Coreflow reconciler wrote (workload, routing, immutable-image, immutable-adoption and immutable-attestation) and every externalGateway record of a route its service no longer declares; a record of a route still declared stays, failed or not, so do the records of other observers, and a rerun removes nothing. The deployment operations get the index service_deployment_operation_newest, which the model builds at start. Cloudly 33.23.0 adds delegated admin temp tokens: a human administrator who signed in within five minutes mints a token for the scopes secrets and serviceCredentials that lives one hour at most (servezone admin temp-token mint, list and revoke); Cloudly stores only its SHA-256, in cloudly_delegated_admin_tokens, records every use, refusal and failed write in cloudly_delegated_admin_token_audit, and a token never writes a reserved name and never deletes. Cloudly 33.23.0 moves to @serve.zone/interfaces 32.61.0 and its dashboard to @design.estate/dees-catalog 23 (controls are 30px tall instead of 32px, dialogs and sheets have 14px corners instead of 20px). With Cloudly 33.22.0 the scheduled backup-all-services task, which 1.29.8 turned on with CLOUDLY_BACKUP_CRON, backs up each workload service on the node and cluster its data lives on: on a Cloudly with more than one cluster every service was refused, and on one with a single cluster every backup was refused backup-node-required, while the run was recorded completed. A service with nothing to back up is skipped, a service without a target fails by name, and a run in which any service failed ends failed and names each one. Cloudly 33.22.0 adds servezone service credential inspect, ensure, rotate and revoke for the service machine credential @serve.zone/platformclient 4.0.0 and later signs in with, and names a refused actor SERVICE_CREDENTIAL_UNAUTHORIZED. Neither release adds an environment variable or a setting, and neither changes the Dockerfile or the qenv.yml requirements, so 1.29.9 keeps 1.29.8's runtime contract, CLOUDLY_BACKUP_CRON 15 2 * * * included, and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: the data step has no way back. Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.29.8, 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. 1.29.9 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.9 still exports the ledger with its documents; 1.30.5 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.9 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.5, and 1.29.9 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2, 1.30.3 or 1.30.4, which sort above it. Like 1.29.8, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.8 the upgrade moves Cloudly from 33.21.4 to 33.23.0 and runs the data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0. From 1.29.7 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.8, the Cloudly 33.21.4 template, and describes the changes of Cloudly 33.21.4 and the releases before it that an upgrade from 1.29.7 or earlier also takes: Move Cloudly to 33.21.4, a fixes-only patch over 33.21.3, on the config of 1.29.7, and turn on Cloudly's scheduled workload backups: 1.29.8 differs from 1.29.7 only in its image, its changelog and the environment variable CLOUDLY_BACKUP_CRON. Cloudly registers its backup-all-services task only when CLOUDLY_BACKUP_CRON is set, and no earlier template set it, so the databases of the workload services Cloudly runs were never backed up on a schedule. This template sets it to 15 2 * * *: every day at 02:15 UTC, clear of Onebox's backup-archive-prune at 03:00, Cloudly creates a backup of every workload service with backup-enabled resources, and it keeps the newest 24 settled backups of each service (CLOUDLY_BACKUP_KEEP_LAST, which the template does not set, defaults to 24) and deletes older ones with their stored archives. These backups cover the workload databases; a workload's object storage is not part of them (AGL-473). The variable is a plain declaration with no secret and no delivery and is not required. A host that upgrades a service gives a key the service does not have yet the template's value, so an upgraded Cloudly runs the schedule; an operator's stored value for the key wins over the template's. Cloudly 33.21.4 derives the owner of a launcher-environment secret file from the identity the workload runs as (AGL-459): every launcher delivery was published 0:0 and 0400, so a workload that runs as another user could not read its secrets and WorkloadInit exited with SECRET_SOURCE. The manifest publication now derives the owner per target cluster from the spec's runAs and the pinned image's config.User with @serve.zone/interfaces 32.60.0 deriveLauncherSecretDeliveryOwner, an owner that does not derive is refused launcher-secret-ownership-unknown, and the manifests 33.21.3 published are published again on upgrade. On @serve.zone/interfaces 32.60.0, external registry answers no longer carry the stored credential, Test connection proves the credentials through the Docker Registry v2 login, and a service update merges only the fields it states, so Edit and Configure Mail work again for a service with a runtime spec or an immutable image. Cloudly 33.21.4 also fixes the dashboard (AGL-387 to AGL-401: the status bar, the fonts, the dropdown presets, the view actions, the Secrets wizard and index, the Logs view and the App Store view), refuses a deployment without an existing service and node, a node action the node can never receive and a Jump code for a cluster without a relay, and bootstraps a node with the pinned Spark 35.1.18 release. Cloudly 33.21.4 adds no data step and keeps the data version 0.32.0, so an upgrade from 1.29.7 runs no data step; it adds no index, no collection and no setting, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is that of 1.29.7 with CLOUDLY_BACKUP_CRON and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and an upgrade from 1.29.6 or earlier runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0, which has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. 1.29.8 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.8 still exports the ledger with its documents; 1.30.4 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.8 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.4, and 1.29.8 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2 or 1.30.3, which sort above it. Like 1.29.7, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.7 the upgrade moves Cloudly from 33.21.3 to 33.21.4 and turns the scheduled workload backups on. From 1.29.6 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.7, the Cloudly 33.21.3 template, and describes the changes of Cloudly 33.21.3 and the releases before it that an upgrade from 1.29.6 or earlier also takes: Move Cloudly to 33.21.3, a patch over 33.21.2 that adds one forward-only data step, on the config of 1.29.6: 1.29.7 differs from 1.29.6 only in its image and changelog. A service migrated from Swarm reaches its object storage at its node's Corestore endpoint again. The Secrets v2 cutover turned the Docker-era Corestore binding environment, the nine object-storage endpoint names and, for a binding with retention, the eleven S3_RETENTION_* names, into service-owned secrets in scope platform:cloudly-corestore, the endpoint names with the Docker host corestore; a Corestore binding now gives the endpoint names as plain environment composed from the node's endpoint, and the secrets the cutover left behind replaced that environment in the workload, since a secret wins over an inherited variable of the same name, so such a service exited at startup with getaddrinfo ENOTFOUND corestore. Cloudly 33.21.3 adds the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. It retires every active service-owned secret in that scope whose key is one of the twenty non-secret object-storage names (the nine endpoint names and the eleven S3_RETENTION_* names) as a lifecycle change retires one: the version is retired and kept, never purged, and the metadata names the step as the author. The service's secrets revision advances once, and its desired manifests are published again without those secrets under a new secret rollout, the previous one accepted for five more minutes, so the runtime replaces the service's replicas once; a single-replica service restarts briefly. A service whose image plan or secret deployment plan the runtime already refuses gets its manifest from its next promotion. The six credential names, other scopes, other owners and greenfield services are left alone; a database binding gives no plain name, so no database secret is touched. Each service is one transaction, so a rerun writes nothing. The step has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. Cloudly 33.21.3 adds no index, no collection, no setting and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.6 and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and every upgrade to this template runs the step to 0.32.0. 1.29.7 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.7 still exports the ledger with its documents; 1.30.3 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.7 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.3, and 1.29.7 is never the upgrade target of 1.30.0, 1.30.1 or 1.30.2, which sort above it. Like 1.29.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.6 the upgrade moves Cloudly from 33.21.2 to 33.21.3 and runs the step to 0.32.0. From 1.29.5 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.6, the Cloudly 33.21.2 template, and describes the changes of Cloudly 33.21.2 and the releases before it that an upgrade from 1.29.5 or earlier also takes: Move Cloudly to 33.21.2, a fixes-only patch over 33.21.1, on the config of 1.29.5: 1.29.6 differs from 1.29.5 only in its image and changelog. A recovery redeploy over a dead retained rollout plan, one that no runtime can execute (such as a Swarm-era plan) and whose operation is settled, is no longer refused PLATFORM_CAPABILITY_UNAVAILABLE by its reserve and promote preflights while its Corestore bindings wait for their credentials: such a service has no plan a runtime can run, so it is issued no credential grant until a promotion writes one, and its database and object-storage bindings could not become ready before the promotion those preflights refused. The reserve and promote preflights now defer the readiness of such a binding while it is requested or provisioning, as the promote preflight of a held service's first promotion already did, and warn PLATFORM_BINDING_READINESS_DEFERRED; a binding that adopts a legacy resource is deferred only while Corestore lists that resource as authorized for it, and the route preflight after the rollout still requires every binding ready. The retained plan is judged by the operation it names, as the promotion fence reads it, never by the service's latest operation. 1.29.6 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.6 still exports the ledger with its documents; 1.30.2 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.6 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.2, and 1.29.6 is never the upgrade target of 1.30.0 or 1.30.1, which sort above it. Cloudly 33.21.2 adds no data step, no index and no setting and keeps the data version 0.31.0, so an upgrade from 1.29.5, 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.5 and requires Onebox 33.8.0. Like 1.29.5, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.5 the upgrade moves Cloudly from 33.21.1 to 33.21.2. From 1.29.4 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5, the Cloudly 33.21.1 template, and describes the changes of Cloudly 33.21.1 and the releases before it that an upgrade from 1.29.4 or earlier also takes: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
+            "breaking": true,
+            "requiresManualReview": true,
+            "migrationRequired": true,
+            "backupBeforeUpgrade": true,
+            "requiresFeatures": [
+              "storage.object-storage.backup-exclusions"
+            ],
+            "healthCheck": {
+              "path": "/status",
+              "port": 3000,
+              "expectedStatus": 200
+            }
+          },
+          "configDigest": "sha256:7d30a5bd549b1aef2b292d00adfb6004e7d41beb0f590f22b3db5e12546f9d7e"
+        },
         {
           "version": "1.30.0",
           "config": {
-            "image": "code.foss.global/serve.zone/cloudly@sha256:cd0bfc82575d9a4a9d0808e19ec61b0bf188d83729d66195afbf5f373164f16e",
+            "image": "code.foss.global/serve.zone/cloudly@sha256:cd0bfc82575d9a4a9d0808e19ec61b0bf188d83729d66195afbf5f373164f16e",
+            "targetPorts": [
+              {
+                "name": "web",
+                "port": 3000,
+                "protocol": "http",
+                "default": true
+              }
+            ],
+            "envVars": [
+              {
+                "key": "SERVEZONE_ENVIRONMENT",
+                "value": "production",
+                "description": "Cloudly runtime environment.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_PUBLIC_ORIGIN",
+                "value": "https://${SERVICE_DOMAIN}",
+                "description": "Canonical public Cloudly origin including protocol and optional public port.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_LISTENPORT",
+                "value": "3000",
+                "description": "Internal Cloudly HTTP port inside the container.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_SSLMODE",
+                "value": "external",
+                "description": "Use external TLS termination through Onebox or dcrouter.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_SECRET_KEYRING_PATH",
+                "value": "/run/secrets/cloudly-secrets-v2-keyring.json",
+                "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_SECRET_KEYRING_FILE",
+                "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.",
+                "required": true,
+                "secret": true,
+                "delivery": {
+                  "type": "file",
+                  "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json",
+                  "uid": 0,
+                  "gid": 0,
+                  "mode": 256
+                }
+              },
+              {
+                "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH",
+                "value": "/run/secrets/cloudly-workloadinit-approval.json",
+                "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE",
+                "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.",
+                "required": true,
+                "secret": true,
+                "delivery": {
+                  "type": "file",
+                  "targetPath": "/run/secrets/cloudly-workloadinit-approval.json",
+                  "uid": 0,
+                  "gid": 0,
+                  "mode": 256
+                }
+              },
+              {
+                "key": "MONGODB_URL",
+                "value": "${MONGODB_URI}",
+                "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "MONGODB_NAME",
+                "value": "${MONGODB_DATABASE}",
+                "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "MONGODB_USER",
+                "value": "${MONGODB_USERNAME}",
+                "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "MONGODB_PASS",
+                "value": "${MONGODB_PASSWORD}",
+                "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_ENDPOINT",
+                "value": "onebox-corestore",
+                "description": "S3 endpoint host provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_PORT",
+                "value": "9000",
+                "description": "S3 endpoint port provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_USESSL",
+                "value": "false",
+                "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.",
+                "required": true
+              },
+              {
+                "key": "S3_BUCKET",
+                "value": "${S3_BUCKET}",
+                "description": "Cloudly's own S3 bucket, provisioned by Onebox Corestore: registry storage and kept workload logs.",
+                "required": true
+              },
+              {
+                "key": "S3_ACCESSKEY",
+                "value": "${S3_ACCESS_KEY}",
+                "description": "S3 access key provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_SECRETKEY",
+                "value": "${S3_SECRET_KEY}",
+                "description": "S3 secret key provisioned by Onebox Corestore.",
+                "required": true
+              }
+            ],
+            "platformRequirements": {
+              "mongodb": true,
+              "s3": true
+            },
+            "backupExclusions": [
+              {
+                "storageRequest": "legacy-s3",
+                "keyPrefixes": [
+                  "runtime-logs/"
+                ],
+                "reason": "Workload logs; a restore brings back state, not past logs."
+              }
+            ],
+            "databaseBackupExclusions": [
+              {
+                "database": "mongodb",
+                "collections": [
+                  "cloudly_runtime_log_attempts",
+                  "cloudly_runtime_log_batches",
+                  "cloudly_runtime_log_captures",
+                  "cloudly_runtime_log_inline_payloads",
+                  "cloudly_runtime_log_node_discards",
+                  "cloudly_runtime_log_queue"
+                ],
+                "reason": "Workload log ledger; a restore brings back state, not past logs."
+              }
+            ],
+            "platformOidc": {
+              "redirectPath": "/auth/oidc/callback",
+              "roles": [
+                {
+                  "id": "admin",
+                  "label": "Administrator",
+                  "description": "Full Cloudly administration for this hosted instance."
+                },
+                {
+                  "id": "user",
+                  "label": "User",
+                  "description": "Standard authenticated Cloudly access for this hosted instance."
+                }
+              ],
+              "environmentVariables": {
+                "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER",
+                "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID",
+                "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET",
+                "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI",
+                "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE"
+              },
+              "clientAuthenticationMethod": "client_secret_basic"
+            },
+            "minOneboxVersion": "33.8.0",
+            "changelog": "Move Cloudly to 33.21.0, a feature release over 33.20.4: Cloudly's runtime log ledger is no longer part of its backups, and a restore keeps the logs of every running workload. This template declares the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, as databaseBackupExclusions of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. The ledger holds most of a long-running Cloudly's database, so its backups shrink to the state they protect. Every Cloudly 33.21.0 start writes the log record of each running attempt that has none, before it serves node requests, so the nodes' next batches are kept again instead of being refused as attempts whose logs Cloudly does not keep; the first batch of each capture after a restore states the numbers the restore lost as a gap. A start whose attempts all have their records writes nothing, and an attempt whose record cannot be written is named in a warning and tried again at the next start. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.4; a host without the object-storage feature keeps 1.28.4. Like 1.29.4, this template declares Cloudly's workload log objects non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3). Cloudly 33.21.0 adds no data step and no index and keeps the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection, and the template requires Onebox 33.8.0. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.0. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
+            "breaking": true,
+            "requiresManualReview": true,
+            "migrationRequired": true,
+            "backupBeforeUpgrade": true,
+            "requiresFeatures": [
+              "storage.database.backup-exclusions",
+              "storage.object-storage.backup-exclusions"
+            ],
+            "healthCheck": {
+              "path": "/status",
+              "port": 3000,
+              "expectedStatus": 200
+            }
+          },
+          "configDigest": "sha256:e97c235d8f27d9210d9828a83a61035a7b42b11cba667826f3c7e3dfb37b813c"
+        },
+        {
+          "version": "1.30.1",
+          "config": {
+            "image": "code.foss.global/serve.zone/cloudly@sha256:1e92473c17522967711e8a82459fd8933b8c53b6bad26d0dba516cb9b1eccb34",
             "targetPorts": [
               {
                 "name": "web",
@@ -9619,7 +10213,7 @@
               "clientAuthenticationMethod": "client_secret_basic"
             },
             "minOneboxVersion": "33.8.0",
-            "changelog": "Move Cloudly to 33.21.0, a feature release over 33.20.4: Cloudly's runtime log ledger is no longer part of its backups, and a restore keeps the logs of every running workload. This template declares the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, as databaseBackupExclusions of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. The ledger holds most of a long-running Cloudly's database, so its backups shrink to the state they protect. Every Cloudly 33.21.0 start writes the log record of each running attempt that has none, before it serves node requests, so the nodes' next batches are kept again instead of being refused as attempts whose logs Cloudly does not keep; the first batch of each capture after a restore states the numbers the restore lost as a gap. A start whose attempts all have their records writes nothing, and an attempt whose record cannot be written is named in a warning and tried again at the next start. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.4; a host without the object-storage feature keeps 1.28.4. Like 1.29.4, this template declares Cloudly's workload log objects non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3). Cloudly 33.21.0 adds no data step and no index and keeps the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection, and the template requires Onebox 33.8.0. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.0. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
+            "changelog": "Declare Cloudly's runtime log ledger non-backup data without moving the image: this template runs the Cloudly 33.21.1 image of 1.29.5 and differs from 1.29.5 only in its changelog and two members. databaseBackupExclusions names the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. Every Cloudly start writes the log record of each running attempt that has none before it serves node requests, so the nodes' next batches are kept again after a restore, and the first batch of each capture states the numbers the restore lost as a gap. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.5; a host without the object-storage feature keeps 1.28.4. From 1.29.5 the upgrade moves the template only: Cloudly runs the same image on the same data and no data step runs. From 1.30.0 it moves Cloudly from 33.21.0 to 33.21.1 and keeps the declaration; no data step runs. From 1.29.4 or earlier the template also gains the declaration and the upgrade takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5 and holds for this template: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
             "breaking": true,
             "requiresManualReview": true,
             "migrationRequired": true,
@@ -9634,12 +10228,12 @@
               "expectedStatus": 200
             }
           },
-          "configDigest": "sha256:e97c235d8f27d9210d9828a83a61035a7b42b11cba667826f3c7e3dfb37b813c"
+          "configDigest": "sha256:7b7e2f3b78eeacd5319afa6d054cca426e3eb719e309540e83d1fd0176303993"
         },
         {
-          "version": "1.30.1",
+          "version": "1.30.2",
           "config": {
-            "image": "code.foss.global/serve.zone/cloudly@sha256:1e92473c17522967711e8a82459fd8933b8c53b6bad26d0dba516cb9b1eccb34",
+            "image": "code.foss.global/serve.zone/cloudly@sha256:f9859647487308daa9bbd136d02c7fafd651f17265feb7a68cf01212cc7695e1",
             "targetPorts": [
               {
                 "name": "web",
@@ -9823,7 +10417,7 @@
               "clientAuthenticationMethod": "client_secret_basic"
             },
             "minOneboxVersion": "33.8.0",
-            "changelog": "Declare Cloudly's runtime log ledger non-backup data without moving the image: this template runs the Cloudly 33.21.1 image of 1.29.5 and differs from 1.29.5 only in its changelog and two members. databaseBackupExclusions names the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. Every Cloudly start writes the log record of each running attempt that has none before it serves node requests, so the nodes' next batches are kept again after a restore, and the first batch of each capture states the numbers the restore lost as a gap. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.5; a host without the object-storage feature keeps 1.28.4. From 1.29.5 the upgrade moves the template only: Cloudly runs the same image on the same data and no data step runs. From 1.30.0 it moves Cloudly from 33.21.0 to 33.21.1 and keeps the declaration; no data step runs. From 1.29.4 or earlier the template also gains the declaration and the upgrade takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5 and holds for this template: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
+            "changelog": "Declare Cloudly's runtime log ledger non-backup data without moving the image: this template runs the Cloudly 33.21.2 image of 1.29.6 and differs from 1.29.6 only in its changelog and two members. databaseBackupExclusions names the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. Every Cloudly start writes the log record of each running attempt that has none before it serves node requests, so the nodes' next batches are kept again after a restore, and the first batch of each capture states the numbers the restore lost as a gap. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.6; a host without the object-storage feature keeps 1.28.4. From 1.29.6 the upgrade moves the template only: Cloudly runs the same image on the same data and no data step runs. From 1.30.1 it moves Cloudly from 33.21.1 to 33.21.2 and keeps the declaration; no data step runs. From 1.30.0 it moves Cloudly from 33.21.0 to 33.21.2 and keeps the declaration; no data step runs. From 1.29.5 or earlier the template also gains the declaration and the upgrade takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.6 and holds for this template: Move Cloudly to 33.21.2, a fixes-only patch over 33.21.1, on the config of 1.29.5: 1.29.6 differs from 1.29.5 only in its image and changelog. A recovery redeploy over a dead retained rollout plan, one that no runtime can execute (such as a Swarm-era plan) and whose operation is settled, is no longer refused PLATFORM_CAPABILITY_UNAVAILABLE by its reserve and promote preflights while its Corestore bindings wait for their credentials: such a service has no plan a runtime can run, so it is issued no credential grant until a promotion writes one, and its database and object-storage bindings could not become ready before the promotion those preflights refused. The reserve and promote preflights now defer the readiness of such a binding while it is requested or provisioning, as the promote preflight of a held service's first promotion already did, and warn PLATFORM_BINDING_READINESS_DEFERRED; a binding that adopts a legacy resource is deferred only while Corestore lists that resource as authorized for it, and the route preflight after the rollout still requires every binding ready. The retained plan is judged by the operation it names, as the promotion fence reads it, never by the service's latest operation. 1.29.6 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.6 still exports the ledger with its documents; 1.30.2 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.6 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.2, and 1.29.6 is never the upgrade target of 1.30.0 or 1.30.1, which sort above it. Cloudly 33.21.2 adds no data step, no index and no setting and keeps the data version 0.31.0, so an upgrade from 1.29.5, 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.5 and requires Onebox 33.8.0. Like 1.29.5, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.5 the upgrade moves Cloudly from 33.21.1 to 33.21.2. From 1.29.4 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5, the Cloudly 33.21.1 template, and describes the changes of Cloudly 33.21.1 and the releases before it that an upgrade from 1.29.4 or earlier also takes: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
             "breaking": true,
             "requiresManualReview": true,
             "migrationRequired": true,
@@ -9838,12 +10432,12 @@
               "expectedStatus": 200
             }
           },
-          "configDigest": "sha256:7b7e2f3b78eeacd5319afa6d054cca426e3eb719e309540e83d1fd0176303993"
+          "configDigest": "sha256:895f2947850dd0b645c4e1cae8a0bf7a64fe67e491c4396abe6ddd8115634abc"
         },
         {
-          "version": "1.30.2",
+          "version": "1.30.3",
           "config": {
-            "image": "code.foss.global/serve.zone/cloudly@sha256:f9859647487308daa9bbd136d02c7fafd651f17265feb7a68cf01212cc7695e1",
+            "image": "code.foss.global/serve.zone/cloudly@sha256:81b959285ed18de92aaba8c6c84c64e1c7ae618af39d3905ef229d3c331807f6",
             "targetPorts": [
               {
                 "name": "web",
@@ -10027,7 +10621,7 @@
               "clientAuthenticationMethod": "client_secret_basic"
             },
             "minOneboxVersion": "33.8.0",
-            "changelog": "Declare Cloudly's runtime log ledger non-backup data without moving the image: this template runs the Cloudly 33.21.2 image of 1.29.6 and differs from 1.29.6 only in its changelog and two members. databaseBackupExclusions names the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. Every Cloudly start writes the log record of each running attempt that has none before it serves node requests, so the nodes' next batches are kept again after a restore, and the first batch of each capture states the numbers the restore lost as a gap. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.6; a host without the object-storage feature keeps 1.28.4. From 1.29.6 the upgrade moves the template only: Cloudly runs the same image on the same data and no data step runs. From 1.30.1 it moves Cloudly from 33.21.1 to 33.21.2 and keeps the declaration; no data step runs. From 1.30.0 it moves Cloudly from 33.21.0 to 33.21.2 and keeps the declaration; no data step runs. From 1.29.5 or earlier the template also gains the declaration and the upgrade takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.6 and holds for this template: Move Cloudly to 33.21.2, a fixes-only patch over 33.21.1, on the config of 1.29.5: 1.29.6 differs from 1.29.5 only in its image and changelog. A recovery redeploy over a dead retained rollout plan, one that no runtime can execute (such as a Swarm-era plan) and whose operation is settled, is no longer refused PLATFORM_CAPABILITY_UNAVAILABLE by its reserve and promote preflights while its Corestore bindings wait for their credentials: such a service has no plan a runtime can run, so it is issued no credential grant until a promotion writes one, and its database and object-storage bindings could not become ready before the promotion those preflights refused. The reserve and promote preflights now defer the readiness of such a binding while it is requested or provisioning, as the promote preflight of a held service's first promotion already did, and warn PLATFORM_BINDING_READINESS_DEFERRED; a binding that adopts a legacy resource is deferred only while Corestore lists that resource as authorized for it, and the route preflight after the rollout still requires every binding ready. The retained plan is judged by the operation it names, as the promotion fence reads it, never by the service's latest operation. 1.29.6 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.6 still exports the ledger with its documents; 1.30.2 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.6 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.2, and 1.29.6 is never the upgrade target of 1.30.0 or 1.30.1, which sort above it. Cloudly 33.21.2 adds no data step, no index and no setting and keeps the data version 0.31.0, so an upgrade from 1.29.5, 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.5 and requires Onebox 33.8.0. Like 1.29.5, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.5 the upgrade moves Cloudly from 33.21.1 to 33.21.2. From 1.29.4 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5, the Cloudly 33.21.1 template, and describes the changes of Cloudly 33.21.1 and the releases before it that an upgrade from 1.29.4 or earlier also takes: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
+            "changelog": "Declare Cloudly's runtime log ledger non-backup data without moving the image: this template runs the Cloudly 33.21.3 image of 1.29.7 and differs from 1.29.7 only in its changelog and two members. databaseBackupExclusions names the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. Every Cloudly start writes the log record of each running attempt that has none before it serves node requests, so the nodes' next batches are kept again after a restore, and the first batch of each capture states the numbers the restore lost as a gap. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.7; a host without the object-storage feature keeps 1.28.4. From 1.29.7 the upgrade moves the template only: Cloudly runs the same image on the same data and no data step runs. From 1.30.2 it moves Cloudly from 33.21.2 to 33.21.3, keeps the declaration and runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. From 1.30.1 it moves Cloudly from 33.21.1 to 33.21.3, keeps the declaration and runs that step. From 1.30.0 it moves Cloudly from 33.21.0 to 33.21.3, keeps the declaration and runs that step. Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. From 1.29.6 or earlier the template also gains the declaration and the upgrade takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.7 and holds for this template: Move Cloudly to 33.21.3, a patch over 33.21.2 that adds one forward-only data step, on the config of 1.29.6: 1.29.7 differs from 1.29.6 only in its image and changelog. A service migrated from Swarm reaches its object storage at its node's Corestore endpoint again. The Secrets v2 cutover turned the Docker-era Corestore binding environment, the nine object-storage endpoint names and, for a binding with retention, the eleven S3_RETENTION_* names, into service-owned secrets in scope platform:cloudly-corestore, the endpoint names with the Docker host corestore; a Corestore binding now gives the endpoint names as plain environment composed from the node's endpoint, and the secrets the cutover left behind replaced that environment in the workload, since a secret wins over an inherited variable of the same name, so such a service exited at startup with getaddrinfo ENOTFOUND corestore. Cloudly 33.21.3 adds the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. It retires every active service-owned secret in that scope whose key is one of the twenty non-secret object-storage names (the nine endpoint names and the eleven S3_RETENTION_* names) as a lifecycle change retires one: the version is retired and kept, never purged, and the metadata names the step as the author. The service's secrets revision advances once, and its desired manifests are published again without those secrets under a new secret rollout, the previous one accepted for five more minutes, so the runtime replaces the service's replicas once; a single-replica service restarts briefly. A service whose image plan or secret deployment plan the runtime already refuses gets its manifest from its next promotion. The six credential names, other scopes, other owners and greenfield services are left alone; a database binding gives no plain name, so no database secret is touched. Each service is one transaction, so a rerun writes nothing. The step has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. Cloudly 33.21.3 adds no index, no collection, no setting and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.6 and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and every upgrade to this template runs the step to 0.32.0. 1.29.7 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.7 still exports the ledger with its documents; 1.30.3 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.7 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.3, and 1.29.7 is never the upgrade target of 1.30.0, 1.30.1 or 1.30.2, which sort above it. Like 1.29.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.6 the upgrade moves Cloudly from 33.21.2 to 33.21.3 and runs the step to 0.32.0. From 1.29.5 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.6, the Cloudly 33.21.2 template, and describes the changes of Cloudly 33.21.2 and the releases before it that an upgrade from 1.29.5 or earlier also takes: Move Cloudly to 33.21.2, a fixes-only patch over 33.21.1, on the config of 1.29.5: 1.29.6 differs from 1.29.5 only in its image and changelog. A recovery redeploy over a dead retained rollout plan, one that no runtime can execute (such as a Swarm-era plan) and whose operation is settled, is no longer refused PLATFORM_CAPABILITY_UNAVAILABLE by its reserve and promote preflights while its Corestore bindings wait for their credentials: such a service has no plan a runtime can run, so it is issued no credential grant until a promotion writes one, and its database and object-storage bindings could not become ready before the promotion those preflights refused. The reserve and promote preflights now defer the readiness of such a binding while it is requested or provisioning, as the promote preflight of a held service's first promotion already did, and warn PLATFORM_BINDING_READINESS_DEFERRED; a binding that adopts a legacy resource is deferred only while Corestore lists that resource as authorized for it, and the route preflight after the rollout still requires every binding ready. The retained plan is judged by the operation it names, as the promotion fence reads it, never by the service's latest operation. 1.29.6 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.6 still exports the ledger with its documents; 1.30.2 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.6 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.2, and 1.29.6 is never the upgrade target of 1.30.0 or 1.30.1, which sort above it. Cloudly 33.21.2 adds no data step, no index and no setting and keeps the data version 0.31.0, so an upgrade from 1.29.5, 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.5 and requires Onebox 33.8.0. Like 1.29.5, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.5 the upgrade moves Cloudly from 33.21.1 to 33.21.2. From 1.29.4 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5, the Cloudly 33.21.1 template, and describes the changes of Cloudly 33.21.1 and the releases before it that an upgrade from 1.29.4 or earlier also takes: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
             "breaking": true,
             "requiresManualReview": true,
             "migrationRequired": true,
@@ -10042,12 +10636,12 @@
               "expectedStatus": 200
             }
           },
-          "configDigest": "sha256:895f2947850dd0b645c4e1cae8a0bf7a64fe67e491c4396abe6ddd8115634abc"
+          "configDigest": "sha256:ff1a4b84dfc7d931752c5c2ab1a8e6f9165c0f87601c88306f8e045c1c79bdd6"
         },
         {
-          "version": "1.30.3",
+          "version": "1.30.4",
           "config": {
-            "image": "code.foss.global/serve.zone/cloudly@sha256:81b959285ed18de92aaba8c6c84c64e1c7ae618af39d3905ef229d3c331807f6",
+            "image": "code.foss.global/serve.zone/cloudly@sha256:e8e0f2b5a9e3d7aff4ba3201e358e99900b5c3792c24873b3dc0b3cb3f921439",
             "targetPorts": [
               {
                 "name": "web",
@@ -10178,6 +10772,12 @@
                 "value": "${S3_SECRET_KEY}",
                 "description": "S3 secret key provisioned by Onebox Corestore.",
                 "required": true
+              },
+              {
+                "key": "CLOUDLY_BACKUP_CRON",
+                "value": "15 2 * * *",
+                "description": "Schedule of Cloudly's backup-all-services task, which backs up every workload service's databases (not its object storage); Cloudly registers the task only while this is set. Daily at 02:15 UTC, clear of Onebox's backup-archive-prune at 03:00.",
+                "required": false
               }
             ],
             "platformRequirements": {
@@ -10231,7 +10831,7 @@
               "clientAuthenticationMethod": "client_secret_basic"
             },
             "minOneboxVersion": "33.8.0",
-            "changelog": "Declare Cloudly's runtime log ledger non-backup data without moving the image: this template runs the Cloudly 33.21.3 image of 1.29.7 and differs from 1.29.7 only in its changelog and two members. databaseBackupExclusions names the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. Every Cloudly start writes the log record of each running attempt that has none before it serves node requests, so the nodes' next batches are kept again after a restore, and the first batch of each capture states the numbers the restore lost as a gap. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.7; a host without the object-storage feature keeps 1.28.4. From 1.29.7 the upgrade moves the template only: Cloudly runs the same image on the same data and no data step runs. From 1.30.2 it moves Cloudly from 33.21.2 to 33.21.3, keeps the declaration and runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. From 1.30.1 it moves Cloudly from 33.21.1 to 33.21.3, keeps the declaration and runs that step. From 1.30.0 it moves Cloudly from 33.21.0 to 33.21.3, keeps the declaration and runs that step. Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. From 1.29.6 or earlier the template also gains the declaration and the upgrade takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.7 and holds for this template: Move Cloudly to 33.21.3, a patch over 33.21.2 that adds one forward-only data step, on the config of 1.29.6: 1.29.7 differs from 1.29.6 only in its image and changelog. A service migrated from Swarm reaches its object storage at its node's Corestore endpoint again. The Secrets v2 cutover turned the Docker-era Corestore binding environment, the nine object-storage endpoint names and, for a binding with retention, the eleven S3_RETENTION_* names, into service-owned secrets in scope platform:cloudly-corestore, the endpoint names with the Docker host corestore; a Corestore binding now gives the endpoint names as plain environment composed from the node's endpoint, and the secrets the cutover left behind replaced that environment in the workload, since a secret wins over an inherited variable of the same name, so such a service exited at startup with getaddrinfo ENOTFOUND corestore. Cloudly 33.21.3 adds the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. It retires every active service-owned secret in that scope whose key is one of the twenty non-secret object-storage names (the nine endpoint names and the eleven S3_RETENTION_* names) as a lifecycle change retires one: the version is retired and kept, never purged, and the metadata names the step as the author. The service's secrets revision advances once, and its desired manifests are published again without those secrets under a new secret rollout, the previous one accepted for five more minutes, so the runtime replaces the service's replicas once; a single-replica service restarts briefly. A service whose image plan or secret deployment plan the runtime already refuses gets its manifest from its next promotion. The six credential names, other scopes, other owners and greenfield services are left alone; a database binding gives no plain name, so no database secret is touched. Each service is one transaction, so a rerun writes nothing. The step has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. Cloudly 33.21.3 adds no index, no collection, no setting and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.6 and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and every upgrade to this template runs the step to 0.32.0. 1.29.7 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.7 still exports the ledger with its documents; 1.30.3 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.7 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.3, and 1.29.7 is never the upgrade target of 1.30.0, 1.30.1 or 1.30.2, which sort above it. Like 1.29.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.6 the upgrade moves Cloudly from 33.21.2 to 33.21.3 and runs the step to 0.32.0. From 1.29.5 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.6, the Cloudly 33.21.2 template, and describes the changes of Cloudly 33.21.2 and the releases before it that an upgrade from 1.29.5 or earlier also takes: Move Cloudly to 33.21.2, a fixes-only patch over 33.21.1, on the config of 1.29.5: 1.29.6 differs from 1.29.5 only in its image and changelog. A recovery redeploy over a dead retained rollout plan, one that no runtime can execute (such as a Swarm-era plan) and whose operation is settled, is no longer refused PLATFORM_CAPABILITY_UNAVAILABLE by its reserve and promote preflights while its Corestore bindings wait for their credentials: such a service has no plan a runtime can run, so it is issued no credential grant until a promotion writes one, and its database and object-storage bindings could not become ready before the promotion those preflights refused. The reserve and promote preflights now defer the readiness of such a binding while it is requested or provisioning, as the promote preflight of a held service's first promotion already did, and warn PLATFORM_BINDING_READINESS_DEFERRED; a binding that adopts a legacy resource is deferred only while Corestore lists that resource as authorized for it, and the route preflight after the rollout still requires every binding ready. The retained plan is judged by the operation it names, as the promotion fence reads it, never by the service's latest operation. 1.29.6 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.6 still exports the ledger with its documents; 1.30.2 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.6 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.2, and 1.29.6 is never the upgrade target of 1.30.0 or 1.30.1, which sort above it. Cloudly 33.21.2 adds no data step, no index and no setting and keeps the data version 0.31.0, so an upgrade from 1.29.5, 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.5 and requires Onebox 33.8.0. Like 1.29.5, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.5 the upgrade moves Cloudly from 33.21.1 to 33.21.2. From 1.29.4 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5, the Cloudly 33.21.1 template, and describes the changes of Cloudly 33.21.1 and the releases before it that an upgrade from 1.29.4 or earlier also takes: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
+            "changelog": "Declare Cloudly's runtime log ledger non-backup data without moving the image: this template runs the Cloudly 33.21.4 image of 1.29.8 and differs from 1.29.8 only in its changelog and two members. databaseBackupExclusions names the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. Every Cloudly start writes the log record of each running attempt that has none before it serves node requests, so the nodes' next batches are kept again after a restore, and the first batch of each capture states the numbers the restore lost as a gap. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.8; a host without the object-storage feature keeps 1.28.4. Like 1.29.8, this template sets CLOUDLY_BACKUP_CRON to 15 2 * * *, so Cloudly backs up the databases of its workload services daily at 02:15 UTC (not their object storage, AGL-473). From 1.29.8 the upgrade moves the template only: Cloudly runs the same image on the same data and no data step runs. From 1.30.3 it moves Cloudly from 33.21.3 to 33.21.4, keeps the declaration, turns the scheduled workload backups on and runs no data step. From 1.30.2 it moves Cloudly from 33.21.2 to 33.21.4, keeps the declaration, turns the scheduled workload backups on and runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. From 1.30.1 it moves Cloudly from 33.21.1 to 33.21.4, keeps the declaration, turns them on and runs that step. From 1.30.0 it moves Cloudly from 33.21.0 to 33.21.4, keeps the declaration, turns them on and runs that step. Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. From 1.29.7 or earlier the template also gains the declaration and the upgrade takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.8 and holds for this template: Move Cloudly to 33.21.4, a fixes-only patch over 33.21.3, on the config of 1.29.7, and turn on Cloudly's scheduled workload backups: 1.29.8 differs from 1.29.7 only in its image, its changelog and the environment variable CLOUDLY_BACKUP_CRON. Cloudly registers its backup-all-services task only when CLOUDLY_BACKUP_CRON is set, and no earlier template set it, so the databases of the workload services Cloudly runs were never backed up on a schedule. This template sets it to 15 2 * * *: every day at 02:15 UTC, clear of Onebox's backup-archive-prune at 03:00, Cloudly creates a backup of every workload service with backup-enabled resources, and it keeps the newest 24 settled backups of each service (CLOUDLY_BACKUP_KEEP_LAST, which the template does not set, defaults to 24) and deletes older ones with their stored archives. These backups cover the workload databases; a workload's object storage is not part of them (AGL-473). The variable is a plain declaration with no secret and no delivery and is not required. A host that upgrades a service gives a key the service does not have yet the template's value, so an upgraded Cloudly runs the schedule; an operator's stored value for the key wins over the template's. Cloudly 33.21.4 derives the owner of a launcher-environment secret file from the identity the workload runs as (AGL-459): every launcher delivery was published 0:0 and 0400, so a workload that runs as another user could not read its secrets and WorkloadInit exited with SECRET_SOURCE. The manifest publication now derives the owner per target cluster from the spec's runAs and the pinned image's config.User with @serve.zone/interfaces 32.60.0 deriveLauncherSecretDeliveryOwner, an owner that does not derive is refused launcher-secret-ownership-unknown, and the manifests 33.21.3 published are published again on upgrade. On @serve.zone/interfaces 32.60.0, external registry answers no longer carry the stored credential, Test connection proves the credentials through the Docker Registry v2 login, and a service update merges only the fields it states, so Edit and Configure Mail work again for a service with a runtime spec or an immutable image. Cloudly 33.21.4 also fixes the dashboard (AGL-387 to AGL-401: the status bar, the fonts, the dropdown presets, the view actions, the Secrets wizard and index, the Logs view and the App Store view), refuses a deployment without an existing service and node, a node action the node can never receive and a Jump code for a cluster without a relay, and bootstraps a node with the pinned Spark 35.1.18 release. Cloudly 33.21.4 adds no data step and keeps the data version 0.32.0, so an upgrade from 1.29.7 runs no data step; it adds no index, no collection and no setting, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is that of 1.29.7 with CLOUDLY_BACKUP_CRON and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and an upgrade from 1.29.6 or earlier runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0, which has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. 1.29.8 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.8 still exports the ledger with its documents; 1.30.4 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.8 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.4, and 1.29.8 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2 or 1.30.3, which sort above it. Like 1.29.7, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.7 the upgrade moves Cloudly from 33.21.3 to 33.21.4 and turns the scheduled workload backups on. From 1.29.6 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.7, the Cloudly 33.21.3 template, and describes the changes of Cloudly 33.21.3 and the releases before it that an upgrade from 1.29.6 or earlier also takes: Move Cloudly to 33.21.3, a patch over 33.21.2 that adds one forward-only data step, on the config of 1.29.6: 1.29.7 differs from 1.29.6 only in its image and changelog. A service migrated from Swarm reaches its object storage at its node's Corestore endpoint again. The Secrets v2 cutover turned the Docker-era Corestore binding environment, the nine object-storage endpoint names and, for a binding with retention, the eleven S3_RETENTION_* names, into service-owned secrets in scope platform:cloudly-corestore, the endpoint names with the Docker host corestore; a Corestore binding now gives the endpoint names as plain environment composed from the node's endpoint, and the secrets the cutover left behind replaced that environment in the workload, since a secret wins over an inherited variable of the same name, so such a service exited at startup with getaddrinfo ENOTFOUND corestore. Cloudly 33.21.3 adds the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. It retires every active service-owned secret in that scope whose key is one of the twenty non-secret object-storage names (the nine endpoint names and the eleven S3_RETENTION_* names) as a lifecycle change retires one: the version is retired and kept, never purged, and the metadata names the step as the author. The service's secrets revision advances once, and its desired manifests are published again without those secrets under a new secret rollout, the previous one accepted for five more minutes, so the runtime replaces the service's replicas once; a single-replica service restarts briefly. A service whose image plan or secret deployment plan the runtime already refuses gets its manifest from its next promotion. The six credential names, other scopes, other owners and greenfield services are left alone; a database binding gives no plain name, so no database secret is touched. Each service is one transaction, so a rerun writes nothing. The step has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. Cloudly 33.21.3 adds no index, no collection, no setting and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.6 and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and every upgrade to this template runs the step to 0.32.0. 1.29.7 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.7 still exports the ledger with its documents; 1.30.3 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.7 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.3, and 1.29.7 is never the upgrade target of 1.30.0, 1.30.1 or 1.30.2, which sort above it. Like 1.29.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.6 the upgrade moves Cloudly from 33.21.2 to 33.21.3 and runs the step to 0.32.0. From 1.29.5 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.6, the Cloudly 33.21.2 template, and describes the changes of Cloudly 33.21.2 and the releases before it that an upgrade from 1.29.5 or earlier also takes: Move Cloudly to 33.21.2, a fixes-only patch over 33.21.1, on the config of 1.29.5: 1.29.6 differs from 1.29.5 only in its image and changelog. A recovery redeploy over a dead retained rollout plan, one that no runtime can execute (such as a Swarm-era plan) and whose operation is settled, is no longer refused PLATFORM_CAPABILITY_UNAVAILABLE by its reserve and promote preflights while its Corestore bindings wait for their credentials: such a service has no plan a runtime can run, so it is issued no credential grant until a promotion writes one, and its database and object-storage bindings could not become ready before the promotion those preflights refused. The reserve and promote preflights now defer the readiness of such a binding while it is requested or provisioning, as the promote preflight of a held service's first promotion already did, and warn PLATFORM_BINDING_READINESS_DEFERRED; a binding that adopts a legacy resource is deferred only while Corestore lists that resource as authorized for it, and the route preflight after the rollout still requires every binding ready. The retained plan is judged by the operation it names, as the promotion fence reads it, never by the service's latest operation. 1.29.6 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.6 still exports the ledger with its documents; 1.30.2 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.6 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.2, and 1.29.6 is never the upgrade target of 1.30.0 or 1.30.1, which sort above it. Cloudly 33.21.2 adds no data step, no index and no setting and keeps the data version 0.31.0, so an upgrade from 1.29.5, 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.5 and requires Onebox 33.8.0. Like 1.29.5, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.5 the upgrade moves Cloudly from 33.21.1 to 33.21.2. From 1.29.4 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5, the Cloudly 33.21.1 template, and describes the changes of Cloudly 33.21.1 and the releases before it that an upgrade from 1.29.4 or earlier also takes: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
             "breaking": true,
             "requiresManualReview": true,
             "migrationRequired": true,
@@ -10246,7 +10846,217 @@
               "expectedStatus": 200
             }
           },
-          "configDigest": "sha256:ff1a4b84dfc7d931752c5c2ab1a8e6f9165c0f87601c88306f8e045c1c79bdd6"
+          "configDigest": "sha256:61778445373bd108eac47584385e2a82558cb742942f0d96d065554c8ad1ac4b"
+        },
+        {
+          "version": "1.30.5",
+          "config": {
+            "image": "code.foss.global/serve.zone/cloudly@sha256:73be2fb57ee6bf40f83569c677f381713a7ddcc38fedbb3f16a7d85fb1e6e33d",
+            "targetPorts": [
+              {
+                "name": "web",
+                "port": 3000,
+                "protocol": "http",
+                "default": true
+              }
+            ],
+            "envVars": [
+              {
+                "key": "SERVEZONE_ENVIRONMENT",
+                "value": "production",
+                "description": "Cloudly runtime environment.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_PUBLIC_ORIGIN",
+                "value": "https://${SERVICE_DOMAIN}",
+                "description": "Canonical public Cloudly origin including protocol and optional public port.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_LISTENPORT",
+                "value": "3000",
+                "description": "Internal Cloudly HTTP port inside the container.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_SSLMODE",
+                "value": "external",
+                "description": "Use external TLS termination through Onebox or dcrouter.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_SECRET_KEYRING_PATH",
+                "value": "/run/secrets/cloudly-secrets-v2-keyring.json",
+                "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_SECRET_KEYRING_FILE",
+                "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.",
+                "required": true,
+                "secret": true,
+                "delivery": {
+                  "type": "file",
+                  "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json",
+                  "uid": 0,
+                  "gid": 0,
+                  "mode": 256
+                }
+              },
+              {
+                "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH",
+                "value": "/run/secrets/cloudly-workloadinit-approval.json",
+                "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE",
+                "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.",
+                "required": true,
+                "secret": true,
+                "delivery": {
+                  "type": "file",
+                  "targetPath": "/run/secrets/cloudly-workloadinit-approval.json",
+                  "uid": 0,
+                  "gid": 0,
+                  "mode": 256
+                }
+              },
+              {
+                "key": "MONGODB_URL",
+                "value": "${MONGODB_URI}",
+                "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "MONGODB_NAME",
+                "value": "${MONGODB_DATABASE}",
+                "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "MONGODB_USER",
+                "value": "${MONGODB_USERNAME}",
+                "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "MONGODB_PASS",
+                "value": "${MONGODB_PASSWORD}",
+                "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_ENDPOINT",
+                "value": "onebox-corestore",
+                "description": "S3 endpoint host provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_PORT",
+                "value": "9000",
+                "description": "S3 endpoint port provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_USESSL",
+                "value": "false",
+                "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.",
+                "required": true
+              },
+              {
+                "key": "S3_BUCKET",
+                "value": "${S3_BUCKET}",
+                "description": "Cloudly's own S3 bucket, provisioned by Onebox Corestore: registry storage and kept workload logs.",
+                "required": true
+              },
+              {
+                "key": "S3_ACCESSKEY",
+                "value": "${S3_ACCESS_KEY}",
+                "description": "S3 access key provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_SECRETKEY",
+                "value": "${S3_SECRET_KEY}",
+                "description": "S3 secret key provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "CLOUDLY_BACKUP_CRON",
+                "value": "15 2 * * *",
+                "description": "Schedule of Cloudly's backup-all-services task, which backs up every workload service's databases (not its object storage); Cloudly registers the task only while this is set. Daily at 02:15 UTC, clear of Onebox's backup-archive-prune at 03:00.",
+                "required": false
+              }
+            ],
+            "platformRequirements": {
+              "mongodb": true,
+              "s3": true
+            },
+            "backupExclusions": [
+              {
+                "storageRequest": "legacy-s3",
+                "keyPrefixes": [
+                  "runtime-logs/"
+                ],
+                "reason": "Workload logs; a restore brings back state, not past logs."
+              }
+            ],
+            "databaseBackupExclusions": [
+              {
+                "database": "mongodb",
+                "collections": [
+                  "cloudly_runtime_log_attempts",
+                  "cloudly_runtime_log_batches",
+                  "cloudly_runtime_log_captures",
+                  "cloudly_runtime_log_inline_payloads",
+                  "cloudly_runtime_log_node_discards",
+                  "cloudly_runtime_log_queue"
+                ],
+                "reason": "Workload log ledger; a restore brings back state, not past logs."
+              }
+            ],
+            "platformOidc": {
+              "redirectPath": "/auth/oidc/callback",
+              "roles": [
+                {
+                  "id": "admin",
+                  "label": "Administrator",
+                  "description": "Full Cloudly administration for this hosted instance."
+                },
+                {
+                  "id": "user",
+                  "label": "User",
+                  "description": "Standard authenticated Cloudly access for this hosted instance."
+                }
+              ],
+              "environmentVariables": {
+                "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER",
+                "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID",
+                "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET",
+                "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI",
+                "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE"
+              },
+              "clientAuthenticationMethod": "client_secret_basic"
+            },
+            "minOneboxVersion": "33.8.0",
+            "changelog": "Declare Cloudly's runtime log ledger non-backup data without moving the image: this template runs the Cloudly 33.23.0 image of 1.29.9 and differs from 1.29.9 only in its changelog and two members. databaseBackupExclusions names the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. Every Cloudly start writes the log record of each running attempt that has none before it serves node requests, so the nodes' next batches are kept again after a restore, and the first batch of each capture states the numbers the restore lost as a gap. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.9; a host without the object-storage feature keeps 1.28.4. Like 1.29.9, this template sets CLOUDLY_BACKUP_CRON to 15 2 * * *, so Cloudly backs up the databases of its workload services daily at 02:15 UTC (not their object storage, AGL-473). From 1.29.9 the upgrade moves the template only: Cloudly runs the same image on the same data and no data step runs. From 1.30.4 it moves Cloudly from 33.21.4 to 33.23.0, keeps the declaration and runs the forward-only data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0. From 1.30.3 it moves Cloudly from 33.21.3 to 33.23.0, keeps the declaration, turns the scheduled workload backups on and runs that step. From 1.30.2 it moves Cloudly from 33.21.2 to 33.23.0, keeps the declaration, turns them on and runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0 and then that step. From 1.30.1 it moves Cloudly from 33.21.1 to 33.23.0 and from 1.30.0 from 33.21.0, keeps the declaration, turns them on and runs both steps. Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. From 1.29.8 or earlier the template also gains the declaration and the upgrade takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.9 and holds for this template: Move Cloudly to 33.23.0 on the config of 1.29.8: 1.29.9 differs from 1.29.8 only in its image and changelog. Cloudly 33.23.0 adds one forward-only data step, retire-stale-reconciliation-stages from 0.32.0 to 0.33.0, and takes the changes of Cloudly 33.22.0, which no template ran. A service's status is now what the producers Cloudly runs state about it: the Pallet runtime's verdict on each target, the current plan's rollout, the newest deployment operation and the records of the observers Cloudly runs (externalGateway, externalGatewayMail, coreMail and platformBinding). The Services list, the Overview and the MCP tool cloudly_list_services read that one status, so a service that showed Failed or Converging for good because of a stage the Swarm-era Coreflow reconciler last reported on 2026-08-03 shows its real state. getServices composes the status for an administrator only; a cluster's relay gets the services without it. The Overview lists every failed service with its reason and a link to it (degraded ones as warnings); its Deployments tile and Recent deployment changes table are gone, and so is the dashboard summary's deployments count. Cloudly no longer serves reportReconciliationStatus and reportDeploymentSnapshots, the Swarm-era Coreflow intake no component sends, and an observer deletes its record when what it observes goes: a route withdrawn from a service and a deleted or tombstoned platform binding withdraw their externalGateway and platformBinding record. The data step deletes, through the database driver before any model is prepared, every reconciliation record of a kind only the Swarm-era Coreflow reconciler wrote (workload, routing, immutable-image, immutable-adoption and immutable-attestation) and every externalGateway record of a route its service no longer declares; a record of a route still declared stays, failed or not, so do the records of other observers, and a rerun removes nothing. The deployment operations get the index service_deployment_operation_newest, which the model builds at start. Cloudly 33.23.0 adds delegated admin temp tokens: a human administrator who signed in within five minutes mints a token for the scopes secrets and serviceCredentials that lives one hour at most (servezone admin temp-token mint, list and revoke); Cloudly stores only its SHA-256, in cloudly_delegated_admin_tokens, records every use, refusal and failed write in cloudly_delegated_admin_token_audit, and a token never writes a reserved name and never deletes. Cloudly 33.23.0 moves to @serve.zone/interfaces 32.61.0 and its dashboard to @design.estate/dees-catalog 23 (controls are 30px tall instead of 32px, dialogs and sheets have 14px corners instead of 20px). With Cloudly 33.22.0 the scheduled backup-all-services task, which 1.29.8 turned on with CLOUDLY_BACKUP_CRON, backs up each workload service on the node and cluster its data lives on: on a Cloudly with more than one cluster every service was refused, and on one with a single cluster every backup was refused backup-node-required, while the run was recorded completed. A service with nothing to back up is skipped, a service without a target fails by name, and a run in which any service failed ends failed and names each one. Cloudly 33.22.0 adds servezone service credential inspect, ensure, rotate and revoke for the service machine credential @serve.zone/platformclient 4.0.0 and later signs in with, and names a refused actor SERVICE_CREDENTIAL_UNAUTHORIZED. Neither release adds an environment variable or a setting, and neither changes the Dockerfile or the qenv.yml requirements, so 1.29.9 keeps 1.29.8's runtime contract, CLOUDLY_BACKUP_CRON 15 2 * * * included, and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: the data step has no way back. Cloudly 33.22.0, 33.21.4 and every Cloudly before them refuse to start on data version 0.33.0, so a return to 1.29.8, 1.30.4 or an earlier template means restoring the database backup taken before the upgrade. 1.29.9 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.9 still exports the ledger with its documents; 1.30.5 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.9 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.5, and 1.29.9 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2, 1.30.3 or 1.30.4, which sort above it. Like 1.29.8, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.8 the upgrade moves Cloudly from 33.21.4 to 33.23.0 and runs the data step retire-stale-reconciliation-stages from 0.32.0 to 0.33.0. From 1.29.7 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.8, the Cloudly 33.21.4 template, and describes the changes of Cloudly 33.21.4 and the releases before it that an upgrade from 1.29.7 or earlier also takes: Move Cloudly to 33.21.4, a fixes-only patch over 33.21.3, on the config of 1.29.7, and turn on Cloudly's scheduled workload backups: 1.29.8 differs from 1.29.7 only in its image, its changelog and the environment variable CLOUDLY_BACKUP_CRON. Cloudly registers its backup-all-services task only when CLOUDLY_BACKUP_CRON is set, and no earlier template set it, so the databases of the workload services Cloudly runs were never backed up on a schedule. This template sets it to 15 2 * * *: every day at 02:15 UTC, clear of Onebox's backup-archive-prune at 03:00, Cloudly creates a backup of every workload service with backup-enabled resources, and it keeps the newest 24 settled backups of each service (CLOUDLY_BACKUP_KEEP_LAST, which the template does not set, defaults to 24) and deletes older ones with their stored archives. These backups cover the workload databases; a workload's object storage is not part of them (AGL-473). The variable is a plain declaration with no secret and no delivery and is not required. A host that upgrades a service gives a key the service does not have yet the template's value, so an upgraded Cloudly runs the schedule; an operator's stored value for the key wins over the template's. Cloudly 33.21.4 derives the owner of a launcher-environment secret file from the identity the workload runs as (AGL-459): every launcher delivery was published 0:0 and 0400, so a workload that runs as another user could not read its secrets and WorkloadInit exited with SECRET_SOURCE. The manifest publication now derives the owner per target cluster from the spec's runAs and the pinned image's config.User with @serve.zone/interfaces 32.60.0 deriveLauncherSecretDeliveryOwner, an owner that does not derive is refused launcher-secret-ownership-unknown, and the manifests 33.21.3 published are published again on upgrade. On @serve.zone/interfaces 32.60.0, external registry answers no longer carry the stored credential, Test connection proves the credentials through the Docker Registry v2 login, and a service update merges only the fields it states, so Edit and Configure Mail work again for a service with a runtime spec or an immutable image. Cloudly 33.21.4 also fixes the dashboard (AGL-387 to AGL-401: the status bar, the fonts, the dropdown presets, the view actions, the Secrets wizard and index, the Logs view and the App Store view), refuses a deployment without an existing service and node, a node action the node can never receive and a Jump code for a cluster without a relay, and bootstraps a node with the pinned Spark 35.1.18 release. Cloudly 33.21.4 adds no data step and keeps the data version 0.32.0, so an upgrade from 1.29.7 runs no data step; it adds no index, no collection and no setting, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is that of 1.29.7 with CLOUDLY_BACKUP_CRON and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and an upgrade from 1.29.6 or earlier runs the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0, which has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. 1.29.8 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.8 still exports the ledger with its documents; 1.30.4 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.8 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.4, and 1.29.8 is never the upgrade target of 1.30.0, 1.30.1, 1.30.2 or 1.30.3, which sort above it. Like 1.29.7, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.7 the upgrade moves Cloudly from 33.21.3 to 33.21.4 and turns the scheduled workload backups on. From 1.29.6 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.7, the Cloudly 33.21.3 template, and describes the changes of Cloudly 33.21.3 and the releases before it that an upgrade from 1.29.6 or earlier also takes: Move Cloudly to 33.21.3, a patch over 33.21.2 that adds one forward-only data step, on the config of 1.29.6: 1.29.7 differs from 1.29.6 only in its image and changelog. A service migrated from Swarm reaches its object storage at its node's Corestore endpoint again. The Secrets v2 cutover turned the Docker-era Corestore binding environment, the nine object-storage endpoint names and, for a binding with retention, the eleven S3_RETENTION_* names, into service-owned secrets in scope platform:cloudly-corestore, the endpoint names with the Docker host corestore; a Corestore binding now gives the endpoint names as plain environment composed from the node's endpoint, and the secrets the cutover left behind replaced that environment in the workload, since a secret wins over an inherited variable of the same name, so such a service exited at startup with getaddrinfo ENOTFOUND corestore. Cloudly 33.21.3 adds the forward-only data step retire-corestore-environment-secrets from 0.31.0 to 0.32.0. It retires every active service-owned secret in that scope whose key is one of the twenty non-secret object-storage names (the nine endpoint names and the eleven S3_RETENTION_* names) as a lifecycle change retires one: the version is retired and kept, never purged, and the metadata names the step as the author. The service's secrets revision advances once, and its desired manifests are published again without those secrets under a new secret rollout, the previous one accepted for five more minutes, so the runtime replaces the service's replicas once; a single-replica service restarts briefly. A service whose image plan or secret deployment plan the runtime already refuses gets its manifest from its next promotion. The six credential names, other scopes, other owners and greenfield services are left alone; a database binding gives no plain name, so no database secret is touched. Each service is one transaction, so a rerun writes nothing. The step has no way back: Cloudly 33.21.2 and every Cloudly before it refuse to start on data version 0.32.0, so a return to 1.29.6, 1.30.2 or an earlier template means restoring the database backup taken before the upgrade. Cloudly 33.21.3 adds no index, no collection, no setting and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.6 and requires Onebox 33.8.0. Keep breaking, manual-review, migration-required and backup-before-upgrade: hosts read the flags from the target version alone, and every upgrade to this template runs the step to 0.32.0. 1.29.7 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.7 still exports the ledger with its documents; 1.30.3 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.7 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.3, and 1.29.7 is never the upgrade target of 1.30.0, 1.30.1 or 1.30.2, which sort above it. Like 1.29.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.6 the upgrade moves Cloudly from 33.21.2 to 33.21.3 and runs the step to 0.32.0. From 1.29.5 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.6, the Cloudly 33.21.2 template, and describes the changes of Cloudly 33.21.2 and the releases before it that an upgrade from 1.29.5 or earlier also takes: Move Cloudly to 33.21.2, a fixes-only patch over 33.21.1, on the config of 1.29.5: 1.29.6 differs from 1.29.5 only in its image and changelog. A recovery redeploy over a dead retained rollout plan, one that no runtime can execute (such as a Swarm-era plan) and whose operation is settled, is no longer refused PLATFORM_CAPABILITY_UNAVAILABLE by its reserve and promote preflights while its Corestore bindings wait for their credentials: such a service has no plan a runtime can run, so it is issued no credential grant until a promotion writes one, and its database and object-storage bindings could not become ready before the promotion those preflights refused. The reserve and promote preflights now defer the readiness of such a binding while it is requested or provisioning, as the promote preflight of a held service's first promotion already did, and warn PLATFORM_BINDING_READINESS_DEFERRED; a binding that adopts a legacy resource is deferred only while Corestore lists that resource as authorized for it, and the route preflight after the rollout still requires every binding ready. The retained plan is judged by the operation it names, as the promotion fence reads it, never by the service's latest operation. 1.29.6 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.6 still exports the ledger with its documents; 1.30.2 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.6 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.2, and 1.29.6 is never the upgrade target of 1.30.0 or 1.30.1, which sort above it. Cloudly 33.21.2 adds no data step, no index and no setting and keeps the data version 0.31.0, so an upgrade from 1.29.5, 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection and no environment variable, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.5 and requires Onebox 33.8.0. Like 1.29.5, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.5 the upgrade moves Cloudly from 33.21.1 to 33.21.2. From 1.29.4 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.5, the Cloudly 33.21.1 template, and describes the changes of Cloudly 33.21.1 and the releases before it that an upgrade from 1.29.4 or earlier also takes: Move Cloudly to 33.21.1, which carries Cloudly 33.21.0 and 33.21.1 over 33.20.4, on the config of 1.29.4: 1.29.5 differs from 1.29.4 only in its image and changelog. Cloudly 33.21.1 is a fixes-only patch over 33.21.0. A greenfield reserve now creates the Corestore binding of each database and object-storage capability it declares, so nobody binds them by hand before the first promotion; a replayed reserve leaves those bindings unchanged, a greenfield cleanup removes the ones never granted together with the service, and a legacy import's successor adopts over them. A Corestore binding of a held service that was never promoted is saved and waits for the first promotion to deliver it, instead of being refused for lack of a target cluster after it was written. The gateway domain sync no longer overwrites the operator's activation of a domain: it writes activationState only when it creates the domain and records the gateway's listing in gatewayListedAt, relay names are minted only in zones the gateway lists and the operator has not ignored, and the relay-namespace guards read every registered zone. A redeploy over a retained rollout plan that no runtime can execute, such as a Swarm-era plan, whose operation is settled supersedes that plan at promotion instead of being refused ROLLOUT_ACTIVE_CONFLICT after its preflight admitted it. A Pallet service that still carries its Swarm-era placement is granted its Corestore credentials again: on a cluster in a Pallet phase its scope is judged by its runtime spec and plan, not by the placement's node name, and every binding the grant computation leaves out is logged with its reason. Cloudly 33.21.0 writes, at every start and before it serves node requests, the log record of each running attempt that has none. 1.29.5 does not declare the runtime log ledger databaseBackupExclusions, so a backup of 1.29.5 still exports the ledger with its documents; 1.30.1 runs the same image and carries that declaration for a host that fulfils storage.database.backup-exclusions. 1.29.5 is the newest Cloudly target of a host that does not fulfil storage.database.backup-exclusions; a host that fulfils it upgrades to 1.30.1, and 1.29.5 is never the upgrade target of 1.30.0, which sorts above it. Cloudly 33.21.0 and 33.21.1 add no data step and no index and keep the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; they add no collection, change neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.4 and requires Onebox 33.8.0. Like 1.29.4, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.1. From 1.29.3 or earlier it also takes the changes the rest of this changelog describes. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
+            "breaking": true,
+            "requiresManualReview": true,
+            "migrationRequired": true,
+            "backupBeforeUpgrade": true,
+            "requiresFeatures": [
+              "storage.database.backup-exclusions",
+              "storage.object-storage.backup-exclusions"
+            ],
+            "healthCheck": {
+              "path": "/status",
+              "port": 3000,
+              "expectedStatus": 200
+            }
+          },
+          "configDigest": "sha256:e30a131ead0cda7dbb4aa799a78ca991d75cf917d9d25abe7c02e604743af19a"
         }
       ]
     },