@serve.zone/appstore 33.10.0 → 33.12.0

package/dist_catalog/catalog.json

@@ -4,9 +4,9 @@
     "major": 1,
     "minor": 0
   },
-  "revision": 33000010000000,
-  "publishedAt": "2026-10-16T14:44:36.000Z",
-  "sourceRevision": "f7c94ffdcf35694d595af5aafe73247b39570b77",
+  "revision": 33000012000000,
+  "publishedAt": "2026-10-17T07:18:39.000Z",
+  "sourceRevision": "12931b30771f0258982cac18513be47a8793547f",
   "apps": [
     {
       "info": {
@@ -59,7 +59,7 @@
           "Docs": "https://serve.zone"
         }
       },
-      "latestVersion": "1.29.3",
+      "latestVersion": "1.30.0",
       "versions": [
         {
           "version": "1.0.0",
@@ -8675,6 +8675,399 @@
             }
           },
           "configDigest": "sha256:f0c221844be3e02d257955a6e356f2755cb289ad01b37c700bc58ae8f11e96c6"
+        },
+        {
+          "version": "1.29.4",
+          "config": {
+            "image": "code.foss.global/serve.zone/cloudly@sha256:89714b0fa1f8aa08c0146c74970f327462ec483a6331c994ea5cb3ab257d1d4c",
+            "targetPorts": [
+              {
+                "name": "web",
+                "port": 3000,
+                "protocol": "http",
+                "default": true
+              }
+            ],
+            "envVars": [
+              {
+                "key": "SERVEZONE_ENVIRONMENT",
+                "value": "production",
+                "description": "Cloudly runtime environment.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_PUBLIC_ORIGIN",
+                "value": "https://${SERVICE_DOMAIN}",
+                "description": "Canonical public Cloudly origin including protocol and optional public port.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_LISTENPORT",
+                "value": "3000",
+                "description": "Internal Cloudly HTTP port inside the container.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_SSLMODE",
+                "value": "external",
+                "description": "Use external TLS termination through Onebox or dcrouter.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_SECRET_KEYRING_PATH",
+                "value": "/run/secrets/cloudly-secrets-v2-keyring.json",
+                "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_SECRET_KEYRING_FILE",
+                "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.",
+                "required": true,
+                "secret": true,
+                "delivery": {
+                  "type": "file",
+                  "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json",
+                  "uid": 0,
+                  "gid": 0,
+                  "mode": 256
+                }
+              },
+              {
+                "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH",
+                "value": "/run/secrets/cloudly-workloadinit-approval.json",
+                "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE",
+                "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.",
+                "required": true,
+                "secret": true,
+                "delivery": {
+                  "type": "file",
+                  "targetPath": "/run/secrets/cloudly-workloadinit-approval.json",
+                  "uid": 0,
+                  "gid": 0,
+                  "mode": 256
+                }
+              },
+              {
+                "key": "MONGODB_URL",
+                "value": "${MONGODB_URI}",
+                "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "MONGODB_NAME",
+                "value": "${MONGODB_DATABASE}",
+                "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "MONGODB_USER",
+                "value": "${MONGODB_USERNAME}",
+                "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "MONGODB_PASS",
+                "value": "${MONGODB_PASSWORD}",
+                "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_ENDPOINT",
+                "value": "onebox-corestore",
+                "description": "S3 endpoint host provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_PORT",
+                "value": "9000",
+                "description": "S3 endpoint port provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_USESSL",
+                "value": "false",
+                "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.",
+                "required": true
+              },
+              {
+                "key": "S3_BUCKET",
+                "value": "${S3_BUCKET}",
+                "description": "Cloudly's own S3 bucket, provisioned by Onebox Corestore: registry storage and kept workload logs.",
+                "required": true
+              },
+              {
+                "key": "S3_ACCESSKEY",
+                "value": "${S3_ACCESS_KEY}",
+                "description": "S3 access key provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_SECRETKEY",
+                "value": "${S3_SECRET_KEY}",
+                "description": "S3 secret key provisioned by Onebox Corestore.",
+                "required": true
+              }
+            ],
+            "platformRequirements": {
+              "mongodb": true,
+              "s3": true
+            },
+            "backupExclusions": [
+              {
+                "storageRequest": "legacy-s3",
+                "keyPrefixes": [
+                  "runtime-logs/"
+                ],
+                "reason": "Workload logs; a restore brings back state, not past logs."
+              }
+            ],
+            "platformOidc": {
+              "redirectPath": "/auth/oidc/callback",
+              "roles": [
+                {
+                  "id": "admin",
+                  "label": "Administrator",
+                  "description": "Full Cloudly administration for this hosted instance."
+                },
+                {
+                  "id": "user",
+                  "label": "User",
+                  "description": "Standard authenticated Cloudly access for this hosted instance."
+                }
+              ],
+              "environmentVariables": {
+                "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER",
+                "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID",
+                "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET",
+                "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI",
+                "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE"
+              },
+              "clientAuthenticationMethod": "client_secret_basic"
+            },
+            "minOneboxVersion": "33.8.0",
+            "changelog": "Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
+            "breaking": true,
+            "requiresManualReview": true,
+            "migrationRequired": true,
+            "backupBeforeUpgrade": true,
+            "requiresFeatures": [
+              "storage.object-storage.backup-exclusions"
+            ],
+            "healthCheck": {
+              "path": "/status",
+              "port": 3000,
+              "expectedStatus": 200
+            }
+          },
+          "configDigest": "sha256:5379de0dd94d6bddcb2541c3e95f70f6868619ab62bfd6e117b5bf7e11e58a2c"
+        },
+        {
+          "version": "1.30.0",
+          "config": {
+            "image": "code.foss.global/serve.zone/cloudly@sha256:cd0bfc82575d9a4a9d0808e19ec61b0bf188d83729d66195afbf5f373164f16e",
+            "targetPorts": [
+              {
+                "name": "web",
+                "port": 3000,
+                "protocol": "http",
+                "default": true
+              }
+            ],
+            "envVars": [
+              {
+                "key": "SERVEZONE_ENVIRONMENT",
+                "value": "production",
+                "description": "Cloudly runtime environment.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_PUBLIC_ORIGIN",
+                "value": "https://${SERVICE_DOMAIN}",
+                "description": "Canonical public Cloudly origin including protocol and optional public port.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_LISTENPORT",
+                "value": "3000",
+                "description": "Internal Cloudly HTTP port inside the container.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_SSLMODE",
+                "value": "external",
+                "description": "Use external TLS termination through Onebox or dcrouter.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_SECRET_KEYRING_PATH",
+                "value": "/run/secrets/cloudly-secrets-v2-keyring.json",
+                "description": "Path to the Cloudly Secrets v2 keyring mounted by Onebox; the keyring value is never placed in the public environment.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_SECRET_KEYRING_FILE",
+                "description": "Existing Cloudly Secrets v2 keyring. Onebox stores it encrypted and mounts it only as a root-readable file; retain the existing keyring when upgrading.",
+                "required": true,
+                "secret": true,
+                "delivery": {
+                  "type": "file",
+                  "targetPath": "/run/secrets/cloudly-secrets-v2-keyring.json",
+                  "uid": 0,
+                  "gid": 0,
+                  "mode": 256
+                }
+              },
+              {
+                "key": "SERVEZONE_WORKLOADINIT_APPROVAL_PATH",
+                "value": "/run/secrets/cloudly-workloadinit-approval.json",
+                "description": "Private WorkloadInit release-approval artifact mounted by Onebox for Coreflow secret-runtime registration.",
+                "required": true
+              },
+              {
+                "key": "SERVEZONE_WORKLOADINIT_APPROVAL_FILE",
+                "description": "Operator-approved WorkloadInit release authority. Onebox stores it encrypted and mounts it only as a root-readable file.",
+                "required": true,
+                "secret": true,
+                "delivery": {
+                  "type": "file",
+                  "targetPath": "/run/secrets/cloudly-workloadinit-approval.json",
+                  "uid": 0,
+                  "gid": 0,
+                  "mode": 256
+                }
+              },
+              {
+                "key": "MONGODB_URL",
+                "value": "${MONGODB_URI}",
+                "description": "Authenticated MongoDB wire-protocol URL provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "MONGODB_NAME",
+                "value": "${MONGODB_DATABASE}",
+                "description": "MongoDB wire-protocol database name provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "MONGODB_USER",
+                "value": "${MONGODB_USERNAME}",
+                "description": "MongoDB wire-protocol username provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "MONGODB_PASS",
+                "value": "${MONGODB_PASSWORD}",
+                "description": "MongoDB wire-protocol password provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_ENDPOINT",
+                "value": "onebox-corestore",
+                "description": "S3 endpoint host provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_PORT",
+                "value": "9000",
+                "description": "S3 endpoint port provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_USESSL",
+                "value": "false",
+                "description": "Use plain HTTP for internal Corestore S3 traffic on the Onebox network.",
+                "required": true
+              },
+              {
+                "key": "S3_BUCKET",
+                "value": "${S3_BUCKET}",
+                "description": "Cloudly's own S3 bucket, provisioned by Onebox Corestore: registry storage and kept workload logs.",
+                "required": true
+              },
+              {
+                "key": "S3_ACCESSKEY",
+                "value": "${S3_ACCESS_KEY}",
+                "description": "S3 access key provisioned by Onebox Corestore.",
+                "required": true
+              },
+              {
+                "key": "S3_SECRETKEY",
+                "value": "${S3_SECRET_KEY}",
+                "description": "S3 secret key provisioned by Onebox Corestore.",
+                "required": true
+              }
+            ],
+            "platformRequirements": {
+              "mongodb": true,
+              "s3": true
+            },
+            "backupExclusions": [
+              {
+                "storageRequest": "legacy-s3",
+                "keyPrefixes": [
+                  "runtime-logs/"
+                ],
+                "reason": "Workload logs; a restore brings back state, not past logs."
+              }
+            ],
+            "databaseBackupExclusions": [
+              {
+                "database": "mongodb",
+                "collections": [
+                  "cloudly_runtime_log_attempts",
+                  "cloudly_runtime_log_batches",
+                  "cloudly_runtime_log_captures",
+                  "cloudly_runtime_log_inline_payloads",
+                  "cloudly_runtime_log_node_discards",
+                  "cloudly_runtime_log_queue"
+                ],
+                "reason": "Workload log ledger; a restore brings back state, not past logs."
+              }
+            ],
+            "platformOidc": {
+              "redirectPath": "/auth/oidc/callback",
+              "roles": [
+                {
+                  "id": "admin",
+                  "label": "Administrator",
+                  "description": "Full Cloudly administration for this hosted instance."
+                },
+                {
+                  "id": "user",
+                  "label": "User",
+                  "description": "Standard authenticated Cloudly access for this hosted instance."
+                }
+              ],
+              "environmentVariables": {
+                "issuerUrl": "SERVEZONE_PLATFORM_OIDC_ISSUER",
+                "clientId": "SERVEZONE_PLATFORM_OIDC_CLIENT_ID",
+                "clientSecret": "SERVEZONE_PLATFORM_OIDC_CLIENT_SECRET",
+                "redirectUri": "SERVEZONE_PLATFORM_OIDC_REDIRECT_URI",
+                "audience": "SERVEZONE_PLATFORM_OIDC_AUDIENCE"
+              },
+              "clientAuthenticationMethod": "client_secret_basic"
+            },
+            "minOneboxVersion": "33.8.0",
+            "changelog": "Move Cloudly to 33.21.0, a feature release over 33.20.4: Cloudly's runtime log ledger is no longer part of its backups, and a restore keeps the logs of every running workload. This template declares the six collections of the ledger, cloudly_runtime_log_attempts, cloudly_runtime_log_batches, cloudly_runtime_log_captures, cloudly_runtime_log_inline_payloads, cloudly_runtime_log_node_discards and cloudly_runtime_log_queue, as databaseBackupExclusions of its MongoDB database: a backup exports them with their indexes and without their documents, so a restore brings them back empty while the slots and the workloads they run stay. The ledger holds most of a long-running Cloudly's database, so its backups shrink to the state they protect. Every Cloudly 33.21.0 start writes the log record of each running attempt that has none, before it serves node requests, so the nodes' next batches are kept again instead of being refused as attempts whose logs Cloudly does not keep; the first batch of each capture after a restore states the numbers the restore lost as a gap. A start whose attempts all have their records writes nothing, and an attempt whose record cannot be written is named in a warning and tried again at the next start. requiresFeatures names storage.database.backup-exclusions beside storage.object-storage.backup-exclusions: a host that does not fulfil the database feature keeps listing this version and refuses it as an install or upgrade target, and its newest Cloudly target stays 1.29.4; a host without the object-storage feature keeps 1.28.4. Like 1.29.4, this template declares Cloudly's workload log objects non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3). Cloudly 33.21.0 adds no data step and no index and keeps the data version 0.31.0, so an upgrade from 1.29.4, 1.29.3 or 1.29.2 runs no data step; it adds no collection, and the template requires Onebox 33.8.0. From 1.29.4 the upgrade moves Cloudly from 33.20.4 to 33.21.0. The rest of this changelog is that of 1.29.4, the Cloudly 33.20.4 template, and describes the changes of Cloudly 33.20.4 and the releases before it that an upgrade from 1.29.3 or earlier also takes: Move Cloudly to 33.20.4, a fixes-only patch over 33.20.3: the runtime log sweep completes again over a large ledger and no longer keeps other controller transactions retrying. Over a ledger of about 140,000 rows every sweep pass failed after about 126 s with the transaction lifetime exceeded, because two of its reads cost the whole ledger rather than their own work: the walk over dropped rows was sorted by id, so it walked the id index over every row to find the few dropped ones, and the orphan listing's held check asked for either state at once, so it read every row of a capture for each of up to 1,000 listed keys in one transaction. Such a step outlived the database's 60 s transaction lifetime and was retried for 120 s, and since every step fences the controller record, other controller transactions write-conflicted for about two of every three minutes and managed VPN credentials missed their 6 s budget at slot starts. Now the sweep's ledger walks are unsorted, the held check reads one state at a time and stops at the first row, and whether a dropped attempt still has rows is probed instead of counted. Every sweep step is one controller transaction of at most 16 rows or keys (was 64; the held check took up to 1,000 keys in one), bounded by a 15 s deadline that includes its retries and commit, so a step that cannot finish ends the pass at once and the next pass continues after the last committed step; before each step the sweep waits for the critical node requests in flight, as the log drain does. No index is added, so the first start builds none. Cloudly 33.20.4 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.3 or 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.3 and requires Onebox 33.8.0. Like 1.29.3, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.3 the upgrade moves Cloudly from 33.20.3 to 33.20.4. From 1.29.2 it also takes the changes of Cloudly 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.3, the Cloudly 33.20.3 template, and describes the changes of Cloudly 33.20.3 and the releases before it that an upgrade from 1.29.2 or earlier also takes: Move Cloudly to 33.20.3, a fixes-only patch over 33.20.2: a service that already holds a Corestore binding of a capability can adopt its own legacy resource of that capability into that binding. Services deployed before Corestore 32 hold a canonical Corestore database and bucket binding created by the generic path, and Corestore (since 34.1.4) refuses any binding beside an unauthorized legacy resource of the capability (legacy-adoption-pending). adoptCorestoreLegacyResources refused such a service binding-exists, a tombstone keeps the service's binding slot and a generic upsert cannot write the adoption, so no path could adopt the resource and the service stayed down. Now, when the legacy owner adopts its own resource (no successorServiceId) and holds a binding of the capability that is enabled, not tombstoned, on the canonical Corestore provider and adopts nothing yet, and the binding with the adoption is valid (a bucket binding names the legacy bucket and its retention intent exactly, a database binding carries neither), the adoption is written onto that binding in the adoption's transaction, a compare-and-set on the binding as it was read; its status returns to requested with its reason cleared, the resource's claim is written, and the plan names the binding's id. Every other binding the adopting service holds still refuses binding-exists: another bucket or retention intent, a database binding with a bucket, another provider, a tombstoned or disabled binding, one adopting another resource, and any binding of a named successor. Adopting again answers the marked binding unchanged. Corestore needs no change: its offline migration reads the plan as for a created binding. Cloudly 33.20.3 adds no data step and keeps the data version 0.31.0, so an upgrade from 1.29.2 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.2 and requires Onebox 33.8.0. Like 1.29.2, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.2 the upgrade moves Cloudly from 33.20.2 to 33.20.3. From 1.29.1 or earlier it also takes the changes of Cloudly 33.20.2, including the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which every Cloudly before 33.20.2 refuses, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade; from 1.29.0 also those of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.2, the Cloudly 33.20.2 template, and describes the changes of Cloudly 33.20.2 and the releases before it that an upgrade from 1.29.1 or earlier also takes: Move Cloudly to 33.20.2, which keeps its registry storage bounded, keeps a running replica when a rollout asks for what it already runs, and no longer loses gateway, mail and CoreMail status while a service lifecycle lease is held; its startup runs the forward-only data step to 0.31.0. A storage retention of the registry runs 3 minutes after start, then daily, or hourly while work remains: before, every tag push recorded an image release that protected its manifests and blobs forever, a greenfield cleanup left its repository behind, blobs no manifest referenced were never collected, and the release rows of deleted services kept protecting their images. A release stays while any service's plan names it or its repository root, any service that is not immutable runs a tag naming its root, a deployment operation still needs it (not terminal, or failed retryably within 7 days) or an unfinished rollout or adoption names it; besides, its service keeps its 3 most recently promoted accepted releases (its rollback targets) and every release observed or promoted within 7 days. Every other release is pruned with the tags naming its root and its image versions, at most 200 a pass. Manifests and blobs nothing reaches any more are deleted once they are older than 24 hours and unused for 24 hours, blobs only when every repository could be judged and Cloudly has served for an hour, at most 20,000 objects a pass; a repository that cannot be read is named in the pass's summary line and skipped, and repositories no service maps to are reported and kept. Every deletion runs in holds of the registry mutation lock that take no new item once 2 seconds passed, so the pushes queued meanwhile go between holds. The per-push prune is gone: after every tagged push it read every repository while holding that lock, and it could delete the child manifests of a multi-platform push in flight. A service deletion deletes only what its own repository owns, its manifests and then its tags, without reading any other repository and without deleting a blob, so a broken repository elsewhere no longer stops it; a greenfield cleanup now retires the provisional service's registry state too, while the operation's reservation holds the repository. The registry (@push.rocks/smartregistry 2.13.0) reads its storage strictly, refuses a manifest PUT that names a blob or child manifest it does not hold with 400 MANIFEST_BLOB_UNKNOWN (a client pushes blobs and children first), collects abandoned upload chunks and serves blob byte ranges, so a layer pull can resume. A rollout, or a retry of one, whose image and configuration a replica already runs keeps that replica running instead of replacing it: whether an incumbent runs a desire is judged by its run's identity or by its content (registry host, repository and image digest, platform, config reference, network attachment, secret authority and storage claims). Deliberately, a spec change that moves only placement or network policy (publicEgress, platformEndpointIds, published ports) no longer restarts a replica, because Pallet applies a new network projection to the running workload in place; upgrading Cloudly restarts no replica. Gateway, mail and CoreMail status reported while a service lifecycle lease is held is no longer refused and dropped after 12 retries: reconciliation status is observation and lives beside the service document, one record per observer in the new collection cloudly_service_reconciliation_stages, the newest report winning, and every API answer that carries a service joins it back as data.reconciliationStatus, so the dashboard reads it as before. Cloudly 33.20.2 adds the forward-only data step service-reconciliation-observations from 0.30.0 to 0.31.0, which moves each readable stage of every service's status into that collection and unsets the status in the service document; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.1 and requires Onebox 33.8.0. The step has no way back: every Cloudly before 33.20.2, 33.20.1 included, refuses to start on data version 0.31.0, so a return to 1.29.1 or an earlier template means restoring the database backup taken before the upgrade, and the registry manifests and blobs the retention deleted by then do not come back with a restored database. Like 1.29.1, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.1 the upgrade moves Cloudly from 33.20.1 to 33.20.2 and runs the step to 0.31.0. From 1.29.0 it also takes the changes of Cloudly 33.20.1; from 1.28.6 or earlier also those of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, also those of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.1, the Cloudly 33.20.1 template, and describes the changes of Cloudly 33.20.1 and the releases before it that an upgrade from 1.29.0 or earlier also takes: Move Cloudly to 33.20.1, a fixes-only patch over 33.20.0: a Cloudly restart no longer blanks a cluster's ingress routes, a revocation withdraws exactly the routes it revokes, and a Pallet registration Cloudly cannot serve for a reason of its own is refused as momentary. After each cold start of Cloudly 33.20.0 the cluster relay applied two route tables of 0 routes before the full one, and every site of the cluster answered 502 for about 2.5 minutes. Two causes, both fixed. First, Cloudly published an empty revocation as its first table of a cluster, because it held no table of it yet; now the first pull or refresh compiles the plan and issues the full table, and a relay's read that comes before it waits up to 6 s for it and is otherwise answered { table: null }, so the relay keeps the table it holds and the full one follows as a push. Second, every node registers again after a restart, which opens a new session, and a route endpoint required an observation reported under the current session, so every route waited until each node reported anew. A reporter is now one node instance under one credential generation (nodeInstanceNonce, Pallet's per-process nonce, stored on every session record): a node instance that only registered again keeps its observations, its cutover uplink endpoint and its cluster ingress registration authority until it reports anew, while a new Pallet process, a new credential generation or a deleted node supersedes them at once. A route's identity names the node instance and credential generation instead of the session id and epoch, so such a registration changes no route. Route sources that cannot be read right after a start keep the relay's table for at most 180 s, logged once per cluster; then one line names the clusters still waiting and their routes are withdrawn as before. A revocation is now the held table less the revoked routes and port routes, issued and pushed at once: before, any revoked route identity made Cloudly publish an empty table for the whole cluster before it fetched certificates for the next one, so one node registering again or one changed route blanked every site of the cluster. A revoked ingress designation still withdraws everything, and route sources that cannot be read still withdraw every route. A Pallet registration that fails for a reason of Cloudly's own (a database connection lost, a write conflict, a race with a credential being applied, a controller not ready yet or any unnamed failure) is answered controller-unavailable and a spent budget controller-deadline, the @serve.zone/interfaces 32.57.0 runtime session refusal contract, so the node waits and registers again instead of ending its session; before, only a spent budget or a stopping controller carried the refusal payload, and the node read every other failure as a verdict. Only the named verdicts carry no payload: an inactive credential, a deleted, retiring (node-retiring) or foreign node, an unverifiable carrier, a replay that states another registration (registration-replay-mismatch), a moved controller identity, an invalid body. A credential check whose store cannot be read now fails instead of reading as a credential that does not verify, so a registration whose carrier Cloudly could not check is momentary too. A request is answered at its budget even when its work ignores the abort signal, and each registration refusal is logged once per node and reason, value-free, until the node's next accepted registration. Cloudly 33.20.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.29.0 runs no data step; it adds no collection, changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.29.0 and requires Onebox 33.8.0. Like 1.29.0, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions; a host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.29.0 the upgrade moves Cloudly from 33.20.0 to 33.20.1. From 1.28.6 or earlier it also takes the changes of Cloudly 33.20.0, including the first start that creates the collection cloudly_runtime_log_inline_payloads and the log ledger rows Cloudly 33.19.2 and earlier refuse, so a return to 1.28.6 or an earlier template needs the backup taken before the upgrade; from 1.28.5 or 1.28.3, which run the Cloudly 33.19.1 image, it also takes the changes of Cloudly 33.19.2, and from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.29.0, the Cloudly 33.20.0 template, and describes the changes of Cloudly 33.20.0 and the releases before it that an upgrade from 1.28.6 or earlier also takes: Move Cloudly to 33.20.0, which serves non-public hostnames to dcrouter alone, writes workload logs as segments and lets a service whose retained rollout plan cannot run be redeployed. A node's cluster ingress serves non-public hostnames, restricted to dcrouter: before, Cloudly compiled only PUBLIC domains into the cluster route table, so a STANDARD hostname got a hub route and a private DNS answer while nothing on the node served it. A domain whose sourceProfileRef is not PUBLIC now compiles, on a moved and on a Pallet-born cluster, into a route whose sourceAllowList is the new setting dcrouterForwardSourceAddresses, the addresses dcrouter's forwarded connections arrive from at a node (set in the dashboard's dcrouter panel or with updateSettings; @serve.zone/interfaces 32.55.1 refuses IPv4-mapped IPv6 prefixes there, write the IPv4 prefix). A change of the setting revokes and reissues the route and refreshes every cluster ingress at once, and the ingress-to-workload grant covers every domain's target port. A non-public domain is left out by name while public routes are still served: while dcrouterForwardSourceAddresses is unset, while the cluster's ingress has not registered route-source-allowlist, and while the live cluster relay speaks @serve.zone/interfaces below 32.55.0. To serve a non-public hostname, the cluster's Coreflow and CoreTraffic must run releases on @serve.zone/interfaces 32.55.0 or later, and CoreTraffic must register the capability. Public port mappings stay public-only. Workload log payloads are written as segments, one object per capture every few minutes instead of one per batch: one object per batch grew Cloudly's bucket by about 7,000 objects an hour. A committed batch keeps its payload inline in the database, in the new collection cloudly_runtime_log_inline_payloads, so it reads as soon as it is committed; the drain then writes a capture's inline payloads as one segment, runtime-logs/<serviceId>/<assignmentId>/<captureId>/<first>-<last>.jsonl, and only after the write moves the ledger rows onto it and removes the inline payloads. A segment is due 5 minutes after its capture's oldest inline payload was committed, at 1 MiB, when the capture's final batch is inline, and early, largest capture first, once all inline payloads pass 16 MiB; at 32 MiB the drain commits nothing more until segments are written, so the queue's bounds refuse new reports with the retryable log-intake-saturated. A stop writes no segment early, and the next start writes the inline payloads when due. Rows committed before this version name their own object as before and age out under the 30-day retention. All payload objects stay below runtime-logs/, the prefix this template declares non-backup. The ledger collection cloudly_runtime_log_batches gains no index, so a start builds none over the existing ledger. A spec-owned service whose retained rollout plan the current runtime cannot execute, such as a Swarm-era promotion written before @serve.zone/interfaces 23.0.0 made the sealed containerInvocation required, can be redeployed: its runtime producer refuses the plan image-plan-missing instead of failing its build step on every pass, and while no deployment operation is in flight the reserve preflight answers GO with the auditable RECOVERY_REDEPLOY_OVER_FAILED_RUNTIME warning, which names the plan and the rules it breaks, instead of NO-GO with CURRENT_RUNTIME_DIGEST_MISSING and RUNTIME_ATTESTATION_MISMATCH. An attested runtime, a mutable current reference, a valid plan over an unattested runtime and an operation in flight still block as before. Cloudly 33.20.0 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.6 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.6 and requires Onebox 33.8.0. Its first start creates the collection cloudly_runtime_log_inline_payloads. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Every workload log batch this version commits names where its payload is, and Cloudly 33.19.2 and earlier refuse such a ledger row whenever they read it, so a return to 1.28.6 or an earlier template after this version committed a log batch requires the backup taken before the upgrade. Like 1.28.5 and 1.28.6, this template declares Cloudly's workload logs non-backup data: backupExclusions names the key prefix runtime-logs/ of Cloudly's own bucket, the object storage platformRequirements.s3 provisions (storage request legacy-s3), and requiresFeatures names storage.object-storage.backup-exclusions. A host that fulfils that feature leaves the objects under runtime-logs/ out of Cloudly's backups, and a restore brings back Cloudly's state, not its past workload logs: it neither restores nor removes objects under that prefix, and Cloudly reports a log batch whose payload it no longer has as a gap. A host that does not fulfil the feature keeps listing this version and refuses it as an install or upgrade target. From 1.28.6 the upgrade moves Cloudly from 33.19.2 to 33.20.0. From 1.28.5, which runs the Cloudly 33.19.1 image of 1.28.3, it also takes the changes of Cloudly 33.19.2; from 1.28.4 or earlier the template also gains the declaration. The rest of this changelog is that of 1.28.4, the Cloudly 33.19.2 template, and describes the changes of Cloudly 33.19.2 and the releases before it that an upgrade from 1.28.5 or earlier also takes: Move Cloudly to 33.19.2, a fixes-only patch over 33.19.1. A greenfield deployment, a legacy import's successor included, proves its namespace blank on the nodes of the Pallet clusters (phase switching or pallet) and nowhere else: a reservation whose service had no rollout plan yet asked every registered node, so a fleet that still keeps a Coreflow-owned Swarm cluster without a relay could never reserve one, and a legacy import's reserve preflight answered NO-GO with CORESTORE_INVENTORY_UNAVAILABLE and NAMESPACE_NOT_PROVEN_BLANK. The deployment preflight and reserveServiceDeployment read the same nodes; a Pallet node whose inventory cannot be read still blocks, a fleet with no Pallet cluster node refuses the reservation CORESTORE_INVENTORY_UNAVAILABLE, and an existing service's preflight still reads its plan's clusters. A greenfield rollout without a Pallet runtime spec is refused GREENFIELD_PALLET_RUNTIME_SPEC_REQUIRED, which is not retryable, instead of falling back to Coreflow placement onto Swarm nodes the namespace proof never covered: state the spec with setServiceRuntimeSpec, then promote again. A rollout with a Pallet spec plans onto the spec's nodes as before. A node's network compilation proves each workload lease once and each protected authority revision and allocation eligibility proof once per transaction (before, every lease was read up to three times per compilation and each read re-proved the same revision and proof, about 24 proof reads per endpoint), so a compilation with 100 endpoints reads 539 records instead of 2,839 (7.8 s instead of 19.8 s on Cloudly's test fixture), and every compilation signs and refuses exactly as before. Cloudly 33.19.2 adds no data step and no collection and keeps the data version 0.30.0, so an upgrade from 1.28.3 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.3 and requires Onebox 33.8.0. From 1.28.2 or earlier, the changes of Cloudly 33.18.3, 33.18.4, 33.18.5, 33.19.0 and 33.19.1 apply too: Runtime log intake no longer holds a node's request while its payload uploads: an admitted reportRuntimeAssignmentLogs costs one append transaction that queues the batch with its payload in the new collection cloudly_runtime_log_queue, and the node is answered once it commits. A background drain writes the payloads, up to 32 at once, and commits the ledger rows and removes the queue rows in coalesced transactions that fence no controller record; every other node request under a request budget counts as critical work, and the drain waits for it at most 1 s per step. A resend of a batch the ledger or the queue already holds is answered replay with nothing written, and while the queue is at its bounds (1024 batches, 64 MiB, 8 concurrent intakes) a report is refused at once with the retryable log-intake-saturated. Object writes list nothing (@lossless.org/client 1.23.3), workload logs are kept for at least 30 days whatever shorter logs.retentionMs a spec states, and one runtime log intake line, logged on its own timer at most once a minute, reports intake latency, queue depth, the drain and refusals. A node that applied its network head is no longer handed a successor that only drops the head's withdrawals, and a successor of the projection a node's receipt names lists only the tombstones its own step retires, so a node with more than 256 retired leases or handoffs stays within the contract's bound. Route verification reads the certificate a route presents on a connection of its own, without pooling or TLS session resumption, so it records the route's subject alternative names; a connection that exposes no certificate fails with TLS_PEER_CERTIFICATE_UNAVAILABLE and a certificate without names with TLS_SUBJECT_ALT_NAMES_MISSING. readServiceDeploymentStatus (@serve.zone/interfaces 32.54.0) reads a deployment operation's status without changing it, and a failed service deletion answers SERVICE_DELETION_FAILED and logs its cause. The first start after 1.28.2 or earlier creates the collection cloudly_runtime_log_queue. On the platform Corestore 32.2.4 (NoSQLDB 10.6.1) a collection creation waits at most 5 s for the open transactions on the Cloudly database, and a transaction an earlier Cloudly left open by exiting before its stop completed (it logs that its shutdown deadline passed) holds that database until its session expires 30 minutes later, so that first start fails with EFENCE_BUSY until then: after an unclean stop of the previous Cloudly, start this version no earlier than 30 minutes after that exit. Cloudly 33.19.1 adds no data step and keeps the data version 0.30.0, so an upgrade from 1.28.2, 1.28.1 or 1.28.0 runs no data step; it changes neither the Dockerfile nor the qenv.yml requirements, and the runtime contract is unchanged from 1.28.2 and requires Onebox 33.8.0. From 1.28.1 or earlier, Cloudly 33.18.2 advances a node's network configuration only once the node's authenticated application receipt names its head. Upgrade every node to Pallet 36.3.0 or later, speaking @serve.zone/interfaces 32.50.0 or later, before this Cloudly update: a node that sends no application receipt, Pallet 36.2.1 or earlier, or whose session speaks @serve.zone/interfaces below 32.50.0, keeps the projection it holds and receives no successor. The receipt wire contract is unchanged. The changes of Cloudly 33.18.2 and the releases before it that an upgrade from 1.28.1 or earlier also takes are described in the changelog of cloudly@1.28.2, which repeats those of every earlier template; this changelog keeps their upgrade path only, because a catalog changelog is bounded at 65536 characters. From 1.2.1 or earlier, startup runs the forward-only data migration from 0.20.0 to 0.21.0 through the step retire-cluster-relay-acme and then the steps runtime-session-protocol-offer to 0.22.0, deployment-machine-user-grants to 0.23.0, corestore-control-token-state to 0.24.0, runtime-log-attempts to 0.25.0, cluster-node-hostname-intent to 0.26.0, deployment-operation-route-evidence to 0.27.0, service-secret-target-predecessor to 0.28.0, route-claim-absent-owners to 0.29.0 and bounded-runtime-history to 0.30.0; retire-cluster-relay-acme drops the relay certificate and DNS collections of Cloudly's retired Cloudflare and Let's Encrypt integration and logs every relay name whose A record Cloudly had written at Cloudflare: remove each such record at the provider. A cluster relay then requires a dcrouter that advertises dns.clientRecords and certificates.exactIssuance, SERVEZONE_SSLMODE=letsencrypt is refused (this template keeps external), and the typed API changes as the Breaking Changes of the Cloudly 33.0.0 changelog list; review them, stop Cloudly and take a verified backup before upgrading, because the 32.9 image refuses the migrated data and rollback requires a full restore. The upgrade is marked migration-required and backup-before-upgrade because hosts read these flags from the target version alone and every upgrade from 1.26.0 or earlier runs the 0.28.0 to 0.30.0 steps, one from 1.24.0 or earlier, and from 1.25.0 whose first start failed, the 0.25.0 to 0.28.0 steps before them, one from 1.19.0 or earlier the 0.24.0 to 0.25.0 step before that, one from 1.15.0 or earlier the 0.23.0 to 0.24.0 step before that, one from 1.8.0 or earlier the 0.22.0 to 0.23.0 step before that, one from 1.6.0 or earlier the 0.21.0 to 0.22.0 step before that and one from 1.2.1 or earlier the migration from 0.20.0 as well, and breaking and manual-review because every Cloudly before 33.17.3 refuses the 0.30.0 data, the 0.30.0 step deletes every archived deployment record, the 0.23.0 step removes stored grants forward-only, Cloudly 33.18.0 ignores APPSTORE_URL and loads the App Store catalog from its catalog registries and, from 1.7.0 or earlier, a username and password deployer, an administrator's deployment reservation and a machine identity or registry bearer issued earlier are refused. An installation on 1.1.10 or earlier also runs the 1.2.0 steps first, from 0.10.0, and requires completed Secrets v2 cutover retirement. Preserve existing user and storage identities, OIDC bindings, the encrypted keyring and WorkloadInit approval as dedicated root-readable files.",
+            "breaking": true,
+            "requiresManualReview": true,
+            "migrationRequired": true,
+            "backupBeforeUpgrade": true,
+            "requiresFeatures": [
+              "storage.database.backup-exclusions",
+              "storage.object-storage.backup-exclusions"
+            ],
+            "healthCheck": {
+              "path": "/status",
+              "port": 3000,
+              "expectedStatus": 200
+            }
+          },
+          "configDigest": "sha256:e97c235d8f27d9210d9828a83a61035a7b42b11cba667826f3c7e3dfb37b813c"
         }
       ]
     },

package/dist_ts_client/storage.d.ts

@@ -13,7 +13,8 @@ export declare function normalizeAppStoreVolumes(configArg: TCatalogVersionConfi
  * The config's storage is judged by the interfaces contract first. Legacy `volumes` become
  * filesystem requests of the reserved `legacyFilesystem` class, `platformRequirements.s3` becomes
  * the `legacy-s3` object-storage request, `backupExclusions` become the declared non-backup key
- * prefixes by request id, and explicit requests and backup exclusions fail closed unless every
+ * prefixes by request id, `databaseBackupExclusions` the collections a backup exports without
+ * documents by database, and explicit requests and backup exclusions fail closed unless every
  * storage feature they need is advertised by the target runtime.
  */
 export declare function normalizeAppStoreStorageConfig(configArg: TCatalogVersionConfig, optionsArg: IAppStoreStorageNormalizationOptions, labelArg?: string): INormalizedAppStoreStorageConfig;