gowalk-cicd 1.0.187 → 1.0.189

package/README.md

@@ -341,7 +341,9 @@ Other factory containers plus the proposed stack may reserve at most 70% of phys
 10% (and 512 MiB) stays available after additional reservations. One-minute load above twice the CPU
 count or full memory pressure of 1% refuses a start. Starts share a host lock (120-second acquisition
 limit); image builds stay parallel. A failed admission emits `backend_runtime_refused` with a fixed
-`gowalk-cicd/backend-runtime-refused.v1 code=…` and leaves existing containers running.
+`gowalk-cicd/backend-runtime-refused.v1 code=…` and leaves existing containers running. A shared host's
+[memory agent](docs/BACKEND_MEMORY.md) keeps running reservations at measured use, and a redeploy is charged only
+for growth beyond the reservations its project already holds.
 
 An operator-managed `/opt/gowalk-backend-runtime/<app>.json`, outside the application mirror, may set
 `schema: "gowalk-cicd/backend-runtime.v1"`, `disabled: true`, `remove_services: ["db"]`, and per-service

package/backend-action/memory_agent.py

@@ -0,0 +1,229 @@
+"""Keep running backend reservations at measured use and compress idle app memory into zswap."""
+import argparse
+import calendar
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import time
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+from native_profile import running_native
+from runtime_host import AGENT_STATE
+from runtime_model import MIB
+
+SCHEMA = "gowalk-cicd/backend-memory.v1"
+APPS = "/opt/gowalk-backends/"
+PROC = Path("/proc")
+CGROUP = Path("/sys/fs/cgroup")
+ZSWAP = Path("/sys/module/zswap/parameters/enabled")
+SETTLE_SECONDS = 600
+DEFAULT, FLOOR, STEP, HYSTERESIS = 64 * MIB, 16 * MIB, 4 * MIB, 8 * MIB
+DATASTORES = ("postgres", "postgis", "timescale", "redis", "valkey", "keydb", "mysql", "mariadb", "mongo",
+              "memcached")
+TEMPLATE = ('{"id":{{json .Id}},"name":{{json .Name}},"running":{{.State.Running}},"pid":{{.State.Pid}},'
+            '"started":{{json .State.StartedAt}},"image":{{json .Config.Image}},"cmd":{{json .Config.Cmd}},'
+            '"labels":{{json .Config.Labels}},"reservation":{{.HostConfig.MemoryReservation}},'
+            '"limit":{{.HostConfig.Memory}}}')
+
+
+def discover():
+    """Every existing container ID (a stopped one keeps its reservation), and the running ones in admission's scope."""
+    try:
+        ids = subprocess.run(["docker", "ps", "-a", "-q", "--no-trunc"], capture_output=True, text=True, timeout=30,
+                             check=True).stdout.split()
+        # Unchecked: a container removed between both calls is only missing from inspect's output.
+        output = subprocess.run(["docker", "inspect", "--format", TEMPLATE, *ids], capture_output=True, text=True,
+                                timeout=60).stdout if ids else ""
+    except (OSError, subprocess.SubprocessError):
+        raise ValueError("docker_unavailable") from None
+    containers = []
+    for line in output.splitlines():
+        container = json.loads(line)
+        labels = container.pop("labels") or {}
+        if container.pop("running") and labels.get("com.docker.compose.project.working_dir", "").startswith(APPS):
+            containers.append(dict(container, project=labels.get("com.docker.compose.project", ""),
+                                   service=labels.get("com.docker.compose.service", "")))
+    return ids, containers
+
+
+def meminfo():
+    pairs = (line.split(":", 1) for line in (PROC / "meminfo").read_text().splitlines())
+    return {key: int(value.split()[0]) * 1024 for key, value in pairs}
+
+
+def cgroup_dir(container):
+    for line in (PROC / str(container["pid"]) / "cgroup").read_text().splitlines():
+        # A reused PID names another cgroup, which is never measured or reclaimed.
+        if line.startswith("0::") and container["id"] in line:
+            return CGROUP / line[3:].lstrip("/")
+    raise FileNotFoundError(container["id"])  # the container's process has exited
+
+
+def measure(directory, store):
+    """The working set, anon and pressure; a datastore's working set also leaves out its active file cache."""
+    stat = dict(line.split() for line in (directory / "memory.stat").read_text().splitlines())
+    cache = int(stat["inactive_file"]) + (int(stat["active_file"]) if store else 0)
+    working_set = int((directory / "memory.current").read_text()) - cache
+    pressure = 0.0
+    for line in (directory / "memory.pressure").read_text().splitlines():
+        if line.startswith("some "):
+            pressure = float(dict(part.split("=") for part in line.split()[1:])["avg60"])
+    return working_set, int(stat["anon"]), pressure
+
+
+def next_reservation(current, working_set, limit):
+    """Working set plus 20% in 4 MiB steps, 16 MiB to the limit; rise at once, fall by 8 MiB or more."""
+    target = max(FLOOR, -(-working_set * 6 // (5 * STEP)) * STEP)
+    target = min(target, limit) if limit > 0 else target
+    return target if not current or target > current or target <= current - HYSTERESIS else current
+
+
+def datastore(container):
+    names = " ".join([container["image"], container["service"],
+                      os.path.basename((container["cmd"] or [""])[0])]).lower()
+    return container["service"].lower() in ("db", "database") or any(word in names for word in DATASTORES)
+
+
+def record(container_id, reservation):
+    """Keep the reservation admission counted before the agent first changes it; a record is never overwritten."""
+    path = AGENT_STATE / container_id
+    if path.exists():
+        return
+    AGENT_STATE.mkdir(mode=0o755, exist_ok=True)
+    descriptor, temporary = tempfile.mkstemp(prefix=".record-", dir=str(AGENT_STATE))
+    try:
+        with os.fdopen(descriptor, "w") as stream:
+            stream.write("%d\n" % reservation)
+            stream.flush()
+            os.fsync(stream.fileno())  # Docker persists the update: a crash must not leave an empty record
+        os.chmod(temporary, 0o644)
+        os.replace(temporary, str(path))
+    except OSError:
+        os.unlink(temporary)
+        raise
+    directory = os.open(str(AGENT_STATE), os.O_RDONLY)
+    try:
+        os.fsync(directory)
+    finally:
+        os.close(directory)
+
+
+def prune(ids):
+    """Forget the records of containers that no longer exist; no other name is ever touched."""
+    for name in os.listdir(str(AGENT_STATE)) if AGENT_STATE.is_dir() else []:
+        if re.fullmatch("[0-9a-f]{64}", name) and name not in ids:
+            (AGENT_STATE / name).unlink()
+
+
+def update_reservation(container_id, reservation):
+    subprocess.run(["docker", "update", "--memory-reservation", str(reservation), container_id],
+                   capture_output=True, timeout=30, check=True)
+
+
+def reclaim(directory, mib):
+    descriptor = os.open(str(directory / "memory.reclaim"), os.O_WRONLY)
+    try:
+        os.write(descriptor, b"%dM" % mib)
+    finally:
+        os.close(descriptor)
+
+
+def handle(container, now, zswap, dry_run):
+    """Measure one container first, then record and update its reservation and reclaim its cold memory."""
+    entry = {"name": container["name"].lstrip("/"), "id": container["id"][:12], "project": container["project"],
+             "service": container["service"]}
+    current, store = container["reservation"], datastore(container)
+    try:
+        started = calendar.timegm(time.strptime(container["started"][:19], "%Y-%m-%dT%H:%M:%S"))
+        if now - started < SETTLE_SECONDS:  # startup imports have not settled yet
+            return dict(entry, settling=True)
+        directory = cgroup_dir(container)
+        working_set, anon, pressure = measure(directory, store)
+    except (FileNotFoundError, ProcessLookupError):
+        return dict(entry, exited=True)
+    except (OSError, ValueError, KeyError):
+        return dict(entry, error="measure_failed")
+    target = next_reservation(current, working_set, container["limit"])
+    if store and target <= (current or DEFAULT):
+        target = current  # datastores only rise: a quiet moment must not cut the cache they refill under load
+    amount = 0 if not zswap or pressure >= 0.5 or store else anon // 4 // MIB
+    entry.update(working_set_mib=working_set // MIB, reservation_mib=current // MIB, target_mib=target // MIB,
+                 reclaim_mib=amount, updated=target != current)
+    if dry_run:
+        return entry
+    if target != current:
+        error = "record_failed"  # admission credits the declared reservation only from this record
+        try:
+            record(container["id"], current or DEFAULT)
+            error = "reservation_update_failed"
+            update_reservation(container["id"], target)
+        except (OSError, subprocess.SubprocessError):
+            entry.update(updated=False, error=error)
+    if amount:
+        try:
+            reclaim(directory, amount)
+        except OSError:  # EAGAIN: the kernel reclaimed less than requested
+            entry["partial"] = True
+    return entry
+
+
+def run(dry_run, now=None):
+    ids, containers = discover()
+    now = time.time() if now is None else now
+    try:
+        zswap = meminfo().get("SwapTotal", 0) > 0 and ZSWAP.read_text().strip() == "Y"
+    except OSError:
+        zswap = False
+    with ThreadPoolExecutor(max_workers=8) as pool:
+        entries = list(pool.map(lambda container: handle(container, now, zswap, dry_run), containers))
+    reserved = sum((entry["target_mib"] * MIB if entry.get("updated") else container["reservation"]) or DEFAULT
+                   for container, entry in zip(containers, entries))
+    if not dry_run:
+        try:
+            prune(set(ids))
+        except OSError:
+            entries.append({"error": "prune_failed"})
+    for entry in entries:
+        if dry_run or "error" in entry:
+            print(json.dumps(entry))
+        elif entry.get("updated"):  # the rollback history
+            print(json.dumps({"name": entry["name"], "id": entry["id"],
+                              "from_mib": entry["reservation_mib"] or DEFAULT // MIB, "to_mib": entry["target_mib"]}))
+    try:
+        natives = sum(unit["HostConfig"]["MemoryReservation"] for unit in running_native())
+    except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError):
+        natives = None
+    memory = meminfo()
+    reserved_mib, cap_mib = (reserved + (natives or 0)) // MIB, int(memory["MemTotal"] * 0.7) // MIB
+
+    def count(key):
+        return sum(1 for entry in entries if entry.get(key))
+
+    return {"schema": SCHEMA, "dry_run": dry_run, "containers": len(containers), "settling": count("settling"),
+            "updated": count("updated"), "partial": count("partial"), "errors": count("error"), "zswap": zswap,
+            "reclaimed_mib": sum(entry.get("reclaim_mib", 0) for entry in entries if not entry.get("partial")),
+            "released_mib": sum((entry["reservation_mib"] or DEFAULT // MIB) - entry["target_mib"]
+                                for entry in entries if entry.get("updated")),
+            "reserved_mib": reserved_mib, "headroom_mib": cap_mib - reserved_mib, "cap_mib": cap_mib,
+            "native_mib": None if natives is None else natives // MIB, "available_mib": memory["MemAvailable"] // MIB,
+            "swap_used_mib": (memory.get("SwapTotal", 0) - memory.get("SwapFree", 0)) // MIB}
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--dry-run", action="store_true", help="print the plan and write nothing")
+    try:
+        summary = run(parser.parse_args(argv).dry_run)
+    except ValueError as error:
+        print(json.dumps({"schema": SCHEMA, "error": str(error)}))
+        return 1
+    print(json.dumps(summary))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/backend-action/native_profile.py

@@ -98,8 +98,10 @@ def running_native():
         for unit, limits in model(profile)['services'].items():
             if states[unit]['ActiveState'] not in ('active', 'activating', 'reloading'):
                 continue
-            reservation = max(int(states[unit]['MemoryLow']), limits['mem_reservation'])
+            # Others count the larger of the applied and declared reservation; the unit's own deploy only the applied.
+            applied = int(states[unit]['MemoryLow'])
+            reservation = max(applied, limits['mem_reservation'])
             containers.append({'Config': {'Labels': {'com.docker.compose.project': path.stem,
                                'com.docker.compose.project.working_dir': str(APPS / path.stem)}},
-                               'HostConfig': {'MemoryReservation': reservation}})
+                               'HostConfig': {'MemoryReservation': reservation}, 'Applied': applied})
     return containers

package/backend-action/runtime_host.py

@@ -1,6 +1,8 @@
 """Admit backend starts against live host headroom and existing container reservations."""
 import json
 import os
+import re
+import stat
 import subprocess
 import time
 import fcntl
@@ -11,6 +13,7 @@ from runtime_model import MIB
 from native_profile import running_native
 
 HOST_POLICY = Path('/etc/gowalk-backend-ssh.json')
+AGENT_STATE = Path("/var/lib/gowalk-backend-memory")
 ROOT_UID = 0
 
 
@@ -63,7 +66,7 @@ def running_containers():
     ids = result.stdout.split()
     if not ids:
         return running_native()
-    template = ('{"Config":{"Labels":{"com.docker.compose.project":'
+    template = ('{"Id":{{json .Id}},"Config":{"Labels":{"com.docker.compose.project":'
                 '{{json (index .Config.Labels "com.docker.compose.project")}},'
                 '"com.docker.compose.project.working_dir":'
                 '{{json (index .Config.Labels "com.docker.compose.project.working_dir")}}}},'
@@ -73,20 +76,40 @@ def running_containers():
     return [json.loads(line) for line in result.stdout.splitlines()] + running_native()
 
 
+def declared(container, live):
+    """The reservation a running container started with, as the memory agent recorded it before resizing."""
+    identifier = str(container.get("Id"))
+    if not re.fullmatch("[0-9a-f]{64}", identifier):
+        return live
+    try:
+        info = (AGENT_STATE / identifier).lstat()
+        if not stat.S_ISREG(info.st_mode) or info.st_uid != ROOT_UID or info.st_mode & 0o022:
+            return live
+        value = int((AGENT_STATE / identifier).read_text())
+    except (OSError, ValueError):
+        return live
+    return value if value > 0 else live
+
+
 def admit(app, model, facts, containers):
     host_enabled()
     required = sum(service["mem_reservation"] for service in model["services"].values())
-    reserved, current = 0, 0
+    reserved, current, held = 0, 0, 0
     for container in containers:
         labels = container.get("Config", {}).get("Labels") or {}
         if not labels.get("com.docker.compose.project.working_dir", "").startswith("/opt/gowalk-backends/"):
             continue
         amount = container["HostConfig"].get("MemoryReservation") or 64 * MIB
         if labels.get("com.docker.compose.project") == app:
+            amount = container.get("Applied", amount)  # a native unit holds its MemoryLow, not the declaration deployed
             current += amount
+            held += declared(container, amount)
         else:
             reserved += amount
-    if reserved + required > facts["total"] * 0.7:
+    # A redeploy is charged only for growth beyond what its project holds: each container at the reservation it
+    # started with (the agent's record), else its live one; an agent rise is never credited to new growth.
+    growth = required - held
+    if growth > 0 and reserved + current + growth > facts["total"] * 0.7:
         raise ValueError("host_reservations_exhausted")
     if facts["available"] - max(0, required - current) < max(512 * MIB, facts["total"] * 0.1):
         raise ValueError("host_memory_exhausted")

package/docs/BACKEND_MEMORY.md

@@ -0,0 +1,85 @@
+# Backend host memory agent
+
+Admission counts reservations (64 MiB when unset), not use, within 70% of RAM. Idle Python APIs use less than
+that default and a third of their memory is cold, so a shared host refused stacks with
+`host_reservations_exhausted` while RAM was still free. `backend-action/memory_agent.py` runs as root every five
+minutes on eight threads. For each running container under `/opt/gowalk-backends/` (the scope admission counts)
+that started at least ten minutes ago, it first measures the working set (`memory.current - inactive_file`), then:
+
+- sets the reservation to the working set plus 20%, rounded up to 4 MiB, at least 16 MiB and at most the
+  container's limit (`docker update --memory-reservation`). App containers rise at once and fall by 8 MiB or more.
+  Datastores (PostgreSQL, PostGIS, TimescaleDB, Redis, Valkey, KeyDB, MySQL, MariaDB, MongoDB or Memcached by
+  image, service or command, or service `db`/`database`) only rise, and only with memory beyond their file cache;
+- while swap is present, zswap is enabled (`/sys/module/zswap/parameters/enabled` is `Y`) and `memory.pressure`
+  `some avg60` is below 0.5, writes a quarter of an app container's anonymous memory to `memory.reclaim`,
+  compressing cold pages into zswap. Datastores are never reclaimed.
+
+Before its first change to a container, the agent records the reservation admission counted (bytes; 64 MiB when
+unset) in `/var/lib/gowalk-backend-memory/<full container id>` (root-owned, 0644, atomic, fsynced, never
+overwritten) and never resizes without it; each real run removes only the 64-hex-named records of containers that
+no longer exist. Admission (`runtime_host.py`) charges a redeploy only for growth beyond what its project holds:
+each running container at its root-owned record (not group- or other-writable) when one exists, otherwise at its
+live reservation, so an agent rise never pays for new growth; each native unit at its applied `MemoryLow`. An
+unchanged or shrinking redeploy is never refused on reservations; a first deploy, an added service or a raised
+declaration needs room under the cap for what exceeds that. The agent never changes limits, CPU, PIDs, images,
+Compose files, runtime profiles or native units.
+
+## Host setup
+
+Install `backend-action/` from the integrity-verified published package as the [SSH entry](BACKEND_SSH_GATE.md)
+describes. Before enabling the timer, move both panel keys' forced commands in `/root/.ssh/authorized_keys` to the
+new release directory: the admission credit lives in its `runtime_host.py`. Add these root-owned units to
+`/etc/systemd/system/`, then create a 16 GiB swap file and enable them:
+
+```ini
+# gowalk-backend-zswap.service
+[Unit]
+DefaultDependencies=no
+Before=swap.target
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/bin/sh -c 'modprobe -q zstd; modprobe -q zsmalloc; cd /sys/module/zswap/parameters && \
+  echo zsmalloc > zpool && echo zstd > compressor && echo 25 > max_pool_percent && \
+  { [ ! -e exclusive_loads ] || echo Y > exclusive_loads; } && echo Y > enabled'
+[Install]
+WantedBy=swap.target
+# gowalk-backend-memory.service
+[Unit]
+After=docker.service
+[Service]
+Type=oneshot
+Nice=10
+ExecStart=/usr/bin/python3 -B /opt/gowalk-cicd-runtime/<version>/backend-action/memory_agent.py
+# gowalk-backend-memory.timer
+[Timer]
+OnBootSec=10min
+OnUnitActiveSec=5min
+[Install]
+WantedBy=timers.target
+```
+
+```sh
+fallocate -l 16G /swapfile && chmod 600 /swapfile && mkswap /swapfile && swapon --priority 10 /swapfile
+echo '/swapfile none swap sw,pri=10 0 0' >> /etc/fstab
+systemctl daemon-reload && systemctl enable --now gowalk-backend-zswap.service gowalk-backend-memory.timer
+```
+
+## Verification and rollback
+
+`systemctl start gowalk-backend-memory.service` runs the agent at once; the oneshot unit never overlaps runs.
+`memory_agent.py --dry-run` prints every container's plan and the summary and writes nothing (no state directory,
+record, prune, `docker update` or reclaim). A real run prints a `{"name","id","from_mib","to_mib"}` history line
+per changed container and a line per error, then the `gowalk-cicd/backend-memory.v1` summary that
+`journalctl _SYSTEMD_UNIT=gowalk-backend-memory.service -n 1 -o cat` shows: `reserved_mib` is the
+admission-equivalent sum after the run, `headroom_mib` what remains of `cap_mib`, `released_mib` what the run
+returned, `zswap` whether reclaim was allowed, `partial` the reclaims completed only in part and `native_mib` null
+(natives excluded) when native discovery failed. Only unavailable Docker exits 1.
+
+Roll back with `systemctl disable --now gowalk-backend-memory.timer && systemctl stop gowalk-backend-memory.service`
+(safe midway: a record precedes each update); Compose restores declared reservations when it recreates containers.
+Restore every recorded value now with this loop, then move both forced commands back to the previous release:
+
+```sh
+for f in /var/lib/gowalk-backend-memory/*; do docker update --memory-reservation "$(cat "$f")" "$(basename "$f")"; done
+```

package/docs/CONFIGURATION.md

@@ -76,6 +76,9 @@ are retained as sparse `workflows.<file>.overrides` entries. An override identif
 a YAML path, with `value` or `remove: true`; step paths use stable IDs/names rather
 than numeric positions. These extensions are app-owned configuration, including
 their execution conditions, dependencies and secret references.
+Reusable workflow jobs may retain GitHub's `secrets: inherit` handoff. This literal is accepted
+only at `jobs.<job>.secrets`; named secret mappings and other credential inputs still require
+GitHub secret references. Sparse overrides are checked at their rendered workflow paths.
 
 Candidate artifacts are enabled automatically for recognized execution. Custom deployment
 commands that the installer cannot establish as candidate-safe retain the normal main

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gowalk-cicd",
-  "version": "1.0.187",
+  "version": "1.0.189",
   "description": "Zero-config GitHub Actions delivery for iOS TestFlight and Android Google Play.",
   "type": "module",
   "bin": {
@@ -41,6 +41,7 @@
     "docs/BACKEND_EDGE.md",
     "docs/BACKEND_SSH_GATE.md",
     "docs/BACKEND_NATIVE.md",
+    "docs/BACKEND_MEMORY.md",
     "docs/CONFIGURATION.md",
     "docs/CI_PERFORMANCE.md",
     "docs/LOCAL_VALIDATION.md"

package/src/config/settings.mjs

@@ -59,8 +59,11 @@ export function assertReferences(value, file, path = []) {
     return;
   }
   if (typeof value !== 'string' || !value.trim()) return;
+  // GitHub's reusable-workflow handoff is syntax, not an inline credential.
+  if (value === 'inherit' && path.length === 3 && path[0] === 'jobs' && path[2] === 'secrets') return;
+  const secretValue = path[0] === 'jobs' && path[2] === 'secrets';
   const key = path.at(-1) || '';
-  if (key === 'runtime-env' && value.includes('\n')) {
+  if (key === 'runtime-env' && value.includes('\n') && !secretValue) {
     for (const line of value.split('\n').filter(line => line.trim() && !line.trim().startsWith('#'))) {
       const match = /^([A-Z_][A-Z0-9_]*)=(.*)$/.exec(line);
       if (!match) throw new MigrationRepair('unsupported_runtime_environment', file, path);
@@ -68,7 +71,7 @@ export function assertReferences(value, file, path = []) {
     }
     return;
   }
-  const credentialKey = /(?:password|private.key|ssh-key|runtime-env|token|secret|credential)/i.test(key);
+  const credentialKey = secretValue || /(?:password|private.key|ssh-key|runtime-env|token|secret|credential)/i.test(key);
   const referencesOnly = /^\s*\$\{\{[^}]+\}\}\s*$/.test(value);
   if ((credentialKey && !referencesOnly) || /-----BEGIN [A-Z ]*PRIVATE KEY-----/.test(value)
       || /https?:\/\/[^\s/@]+:[^\s/@]+@/.test(value)) {

package/src/config/workflows.mjs

@@ -108,8 +108,9 @@ export function planWorkflows(root, templateDir, files, version) {
     delete state.lock.workflows[file];
   }
   writes.push(...removed.map(file => ({ file: `.github/workflows/${file}`, content: null })));
-  assertReferences(state.config, CONFIG);
-  for (const [file, entry] of Object.entries(state.config.workflows)) {
+  const { workflows, ...settings } = state.config;
+  assertReferences(settings, CONFIG);
+  for (const [file, entry] of Object.entries(workflows)) {
     for (const change of entry.overrides) assertReferences(change.value, file, change.path);
   }
   writes.push(...planValidation(root, [...selected, WARM, RECOVERY]));