@hsuite/smart-engines-cli 1.5.1 → 1.6.0

package/dist/_lib/registry-fanout.js

@@ -33,13 +33,20 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_PROBE_TIMEOUT_MS = void 0;
+exports.withRequestTimeout = withRequestTimeout;
 exports.parseAuthChallenge = parseAuthChallenge;
 exports.resolveClusterEndpoints = resolveClusterEndpoints;
 exports.defaultTransport = defaultTransport;
 exports.ensureImageOnClusters = ensureImageOnClusters;
 exports.resolveRegistryDigest = resolveRegistryDigest;
+exports.probeServedDigests = probeServedDigests;
+exports.registryHostOf = registryHostOf;
 exports.resolveAttestationDigest = resolveAttestationDigest;
+exports.assertTagFreeOnMesh = assertTagFreeOnMesh;
 exports.fanOutPushedImage = fanOutPushedImage;
+exports.resolvePushOrigin = resolvePushOrigin;
+exports.requirePushOrigin = requirePushOrigin;
 /**
  * Synchronous Registry-v2 fan-out: copy one already-pushed image to EVERY
  * federated Harbor cluster, then prove it landed on all of them.
@@ -86,6 +93,7 @@ exports.fanOutPushedImage = fanOutPushedImage;
  * never logged.
  */
 const https = __importStar(require("https"));
+const net = __importStar(require("net"));
 const buffer_1 = require("buffer");
 const dns_1 = require("dns");
 /** Media types that identify a multi-arch manifest list / image index (unsupported). */
@@ -117,6 +125,28 @@ const DEPLOYER_PARITY_MANIFEST_ACCEPT = [
     'application/vnd.docker.distribution.manifest.v2+json',
     'application/vnd.docker.distribution.manifest.list.v2+json',
 ].join(', ');
+/** Default per-request wall-clock budget for registry / cluster probes. */
+exports.DEFAULT_PROBE_TIMEOUT_MS = 10_000;
+/**
+ * Reject after `ms` instead of waiting on a request that may never settle.
+ *
+ * {@link defaultTransport} builds a bare `https.request` with no socket timeout,
+ * so a peer that completes the TCP handshake and then black-holes the response
+ * hangs its promise forever. Every probe that fans out across the cluster set
+ * wraps its requests in this.
+ */
+function withRequestTimeout(p, ms, what) {
+    return new Promise((resolveValue, reject) => {
+        const timer = setTimeout(() => reject(new Error(`timed out after ${ms}ms: ${what}`)), ms);
+        p.then((v) => {
+            clearTimeout(timer);
+            resolveValue(v);
+        }, (e) => {
+            clearTimeout(timer);
+            reject(e);
+        });
+    });
+}
 /**
  * Parse a `Bearer realm="...",service="...",scope="..."` challenge. Pure.
  *
@@ -430,19 +460,42 @@ function appendDigestQuery(path, digest) {
  * deployer still re-verifies).
  */
 async function resolveRegistryDigest(opts) {
+    const answers = await probeServedDigests(opts);
+    // Legacy first-responder read. Correct ONLY while the mesh agrees about the
+    // tag; {@link resolveAttestationDigest} is the caller that has to reason
+    // about disagreement (#2787) and uses {@link probeServedDigests} directly.
+    for (const a of answers) {
+        if (a.digest)
+            return a.digest;
+    }
+    return null;
+}
+/**
+ * Ask EVERY cluster endpoint what it serves for `<repo>:<tag>` and return all
+ * the answers, in endpoint order.
+ *
+ * The federated registry name is round-robin over N INDEPENDENT Harbors, so
+ * "what does the registry serve for this tag" is a set of answers, not one
+ * answer. In the window between a `docker push` (which lands on exactly ONE
+ * cluster) and the fan-out, a re-used tag legitimately resolves to the NEW
+ * artifact on the origin and the PREVIOUS artifact on its peers. Collapsing
+ * that set to "whoever answered first" is what signed a stale digest in #2787.
+ */
+async function probeServedDigests(opts) {
     const transport = opts.transport ?? defaultTransport(opts.host);
     const endpoints = await resolveClusterEndpoints(opts.host, opts.resolve4).catch(() => [opts.host]);
-    // Any single endpoint is authoritative for the tag→digest mapping (the
-    // fan-out proves every cluster ends up serving that same digest). Try each
-    // until one answers, so a single-cluster transient miss doesn't fail-soft.
     const client = makeClient(transport, opts.username, opts.password);
     const pullScope = `repository:${opts.repository}:pull`;
     const manifestPath = `/v2/${opts.repository}/manifests/${opts.tag}`;
-    for (const ip of endpoints.length > 0 ? endpoints : [opts.host]) {
+    const timeoutMs = opts.requestTimeoutMs ?? exports.DEFAULT_PROBE_TIMEOUT_MS;
+    // PARALLEL + per-request timeout: one wedged endpoint must not stall the
+    // others, and it must not stall the deploy at all. `Promise.all` preserves
+    // endpoint order, which the origin lookup and the error messages rely on.
+    return Promise.all((endpoints.length > 0 ? endpoints : [opts.host]).map(async (ip) => {
         for (const method of ['HEAD', 'GET']) {
             let res;
             try {
-                res = await client.send({ ip, method, path: manifestPath, headers: { Accept: DEPLOYER_PARITY_MANIFEST_ACCEPT } }, pullScope);
+                res = await withRequestTimeout(client.send({ ip, method, path: manifestPath, headers: { Accept: DEPLOYER_PARITY_MANIFEST_ACCEPT } }, pullScope), timeoutMs, `${method} ${manifestPath} on ${ip}`);
             }
             catch {
                 continue;
@@ -450,29 +503,74 @@ async function resolveRegistryDigest(opts) {
             if (res.status !== 200)
                 continue;
             const served = header(res, 'docker-content-digest');
-            if (served && /^sha256:[0-9a-f]{64}$/.test(served))
-                return served;
+            if (served && /^sha256:[0-9a-f]{64}$/.test(served)) {
+                return { ip, digest: served };
+            }
         }
-    }
-    return null;
+        return { ip, digest: null };
+    }));
+}
+/**
+ * Normalise a `registry.server` into the bare DNS name the endpoint resolvers
+ * and the push probe work with: scheme stripped, port stripped. One definition —
+ * this pair of regexes had accumulated four copies across three files, and a
+ * drift between them would silently change which host the fan-out resolves
+ * versus which one the push is pinned to.
+ */
+function registryHostOf(server) {
+    return server.replace(/^https?:\/\//, '').replace(/:\d+$/, '');
 }
 /**
  * Decide the digest to ATTEST (and to feed the fan-out): the digest Harbor
- * SERVES for `<repo>:<tag>`, resolved the same way the deployer does. Falls back
- * to the local `docker push` summary digest ONLY if the registry can't be probed
- * (offline / dev registry / transient) — a fail-SOFT, never a fail-open, because
- * the deployer independently re-verifies the served digest against the attested
- * one before it pins the pod.
- *
- * See {@link resolveRegistryDigest} for why the served digest — not the push
- * summary — is authoritative for the #1445 strict-equality gate (issue #1589).
+ * SERVES for `<repo>:<tag>`, resolved the same way the deployer does.
+ *
+ * ## Which Harbor, though (#2787)
+ *
+ * `harbor.<domain>` is round-robin over N INDEPENDENT Harbors and a `docker
+ * push` lands on exactly ONE of them. Between the push and the fan-out, a
+ * re-used tag therefore resolves to the NEW artifact on the push origin and to
+ * the PREVIOUS artifact on its peers — both are "the registry-served digest".
+ * This function used to take whichever endpoint answered first, which on
+ * testnet signed `d889ff2a…` (a peer's stale `v12`) for a run that had just
+ * pushed `d6ae4a80…`. The deployer that received that attestation refused the
+ * image — correctly, it cannot tell a stale attestation from a swapped one —
+ * and stayed pinned to the old release while its two peers moved on.
+ *
+ * So:
+ *   - **`originHost` set** (a pinned push landed there) → the result is ALWAYS
+ *     {@link ResolveAttestationDigestOpts.pushDigest}, the digest `buildAndPush`
+ *     read back from the origin registry after the upload. The origin's live
+ *     answer is only a cross-check: if it answers something else we THROW (the
+ *     tag moved underneath us and either value would be a guess), and if it does
+ *     not answer at all we still attest `pushDigest` rather than fall through to
+ *     the peers — the silent-origin case is exactly when the peers unanimously
+ *     serve the PREVIOUS artifact. Peers serving something else are logged and
+ *     left to the fan-out, which copies this manifest to them and re-verifies
+ *     (throwing if it can't).
+ *   - **no `originHost`** (`--skip-push` / unpinned push) → no endpoint is
+ *     privileged, so the endpoints that answer MUST agree. A split mesh THROWS,
+ *     naming every endpoint and its digest — #2766 is a live condition here and
+ *     a coin flip is not an attestation. A served digest may only OVERRIDE
+ *     `pushDigest` when the WHOLE endpoint set answered; a lone answer from a
+ *     partially-reachable mesh also THROWS, because the fan-out would then copy
+ *     that one endpoint's bytes everywhere and make it mesh truth.
+ *
+ * Falls back to `pushDigest` without complaint when NO endpoint can be probed
+ * (offline / dev registry / transient) — a fail-SOFT, never a fail-open: the
+ * deployer independently re-verifies the served digest against the attested one
+ * before it pins the pod.
+ *
+ * See {@link resolveRegistryDigest} for why a registry-served digest — not the
+ * `docker push` summary — is authoritative for the #1445 strict-equality gate
+ * (issue #1589).
  */
 async function resolveAttestationDigest(opts) {
     const log = opts.log ?? (() => undefined);
-    const host = opts.registry.server.replace(/^https?:\/\//, '').replace(/:\d+$/, '');
-    let served = null;
+    const host = registryHostOf(opts.registry.server);
+    const ref = `${opts.registry.repository}:${opts.tag}`;
+    let answers = [];
     try {
-        served = await resolveRegistryDigest({
+        answers = await probeServedDigests({
             host,
             repository: opts.registry.repository,
             tag: opts.tag,
@@ -480,23 +578,159 @@ async function resolveAttestationDigest(opts) {
             password: opts.registry.password,
             transport: opts.transport,
             resolve4: opts.resolve4,
+            ...(opts.requestTimeoutMs !== undefined ? { requestTimeoutMs: opts.requestTimeoutMs } : {}),
         });
     }
     catch (err) {
         log(`  attest: registry digest probe threw (${err.message}); ` +
-            'attesting the local push digest');
+            'attesting the pushed digest');
+    }
+    const originIp = opts.originHost
+        ? await resolveOriginIp(opts.originHost, opts.resolve4).catch(() => null)
+        : null;
+    const originAnswer = originIp ? answers.find((a) => a.ip === originIp) : undefined;
+    // ── Pinned push: only the origin can confirm, and a peer can never override.
+    //
+    // This branch ALWAYS returns `pushDigest` — the value `buildAndPush` resolved
+    // from the origin registry after the upload. The origin's answer is a
+    // cross-check (disagreement ⇒ throw), not an alternative source, and a peer's
+    // answer is never a candidate at all. Falling back to "mesh agreement" when
+    // the origin happens not to answer would re-open the bug through a second
+    // door: on a re-used tag the peers unanimously serve the PREVIOUS artifact
+    // until the fan-out runs, which is precisely the digest #2787 signed.
+    if (opts.originHost) {
+        if (originAnswer?.digest && originAnswer.digest !== opts.pushDigest) {
+            throw new Error(`attest: the push origin ${opts.originHost} (${originAnswer.ip}) serves ` +
+                `${originAnswer.digest} for ${ref}, but this run pushed ${opts.pushDigest}. ` +
+                'The tag moved at the origin between the push and the attestation — refusing to ' +
+                'sign a digest this run did not produce (#2787).');
+        }
+        if (!originAnswer?.digest) {
+            log(`  attest: push origin ${opts.originHost} did not answer for ${ref}; attesting the ` +
+                "digest the push produced there (never a peer's answer for a re-used tag, #2787)");
+        }
+        for (const d of answers.filter((a) => a.digest && a.digest !== opts.pushDigest)) {
+            log(`  attest: peer ${d.ip} still serves ${d.digest} for ${ref} while this run pushed ` +
+                `${opts.pushDigest} at ${opts.originHost} — attesting the ORIGIN digest; the ` +
+                'fan-out converges the peers next and re-verifies every one of them (#2787)');
+        }
+        return opts.pushDigest;
+    }
+    // ── No trusted origin: the endpoints that answer must agree. ────────────
+    const answered = answers.filter((a) => a.digest !== null);
+    const distinct = Array.from(new Set(answered.map((a) => a.digest)));
+    if (distinct.length > 1) {
+        throw new Error(`attest: the cluster registries disagree about ${ref} — ` +
+            answered.map((a) => `${a.ip}=${a.digest}`).join(', ') +
+            `; this run pushed ${opts.pushDigest}. Refusing to sign one cluster's answer at ` +
+            'random (#2787/#2766). Re-run the push so a single origin is authoritative, or ' +
+            'converge the tag across the mesh first.');
     }
-    if (!served) {
-        log(`  attest: could not resolve the registry-served digest for ${opts.registry.repository}:${opts.tag}; ` +
-            'attesting the local push digest (the deployer re-verifies)');
+    if (distinct.length === 0) {
+        log(`  attest: could not resolve the registry-served digest for ${ref}; ` +
+            'attesting the pushed digest (the deployer re-verifies)');
         return opts.pushDigest;
     }
+    const served = distinct[0];
     if (served !== opts.pushDigest) {
-        log(`  attest: registry serves ${served} for ${opts.registry.repository}:${opts.tag} ` +
-            `(local push reported ${opts.pushDigest}); attesting the registry-served digest`);
+        // A served digest may only OVERRIDE what this run pushed when the whole
+        // endpoint set corroborates it (#1589's containerd re-serialization case,
+        // and any single-target registry). If some endpoints stayed silent, "the
+        // endpoints that answer agree" is just "whoever happened to be reachable" —
+        // and the fan-out then makes that lone answer mesh truth by copying its
+        // bytes everywhere, past the point where the deployer's strict-equality
+        // re-check could still catch it.
+        if (answered.length < answers.length) {
+            throw new Error(`attest: only ${answered.length} of ${answers.length} cluster registries answered for ` +
+                `${ref} (` +
+                answered.map((a) => `${a.ip}=${a.digest}`).join(', ') +
+                `), and that answer disagrees with the ${opts.pushDigest} this run pushed. ` +
+                'Refusing to sign an uncorroborated digest from a partially-reachable mesh — ' +
+                're-run once every cluster registry is reachable (#2787).');
+        }
+        log(`  attest: registry serves ${served} for ${ref} ` +
+            `(push reported ${opts.pushDigest}); attesting the registry-served digest`);
     }
     return served;
 }
+/**
+ * Resolve the pinned push-origin hostname to the ONE cluster-ingress address it
+ * names. `resolvePushOrigin` already refused to return a name that resolves to
+ * anything other than exactly one address (its guard 3), so a multi-address
+ * answer here means DNS changed underneath us — treat it as "no origin" rather
+ * than picking one.
+ */
+async function resolveOriginIp(originHost, resolve4) {
+    const ips = await (resolve4 ?? ((h) => dns_1.promises.resolve4(h)))(originHost);
+    return ips.length === 1 ? ips[0] : null;
+}
+/**
+ * #2766 — BEFORE a `docker push`, prove the candidate tag is unoccupied on
+ * EVERY cluster Harbor of the mesh. Tag immutability is enforced per-cluster,
+ * so it cannot prevent divergence BETWEEN registries: a tag that already
+ * exists on any one of them (at any digest — the about-to-be-pushed build has
+ * a fresh, unknowable digest) makes the deploy 412 mid-fan-out at best, and at
+ * worst leaves the tag permanently inconsistent across the mesh (sn1/sn2 at
+ * one digest, sn3 at another, all immutable — unrecoverable without operator
+ * action). This check moves that failure BEFORE the build/push, where the
+ * operator just picks a fresh tag, and lists the divergence it found.
+ *
+ * Semantics:
+ *   - 0/1 endpoint, or DNS that does not fan out → `null` (single-target
+ *     registry; a single origin's own immutability already protects it).
+ *   - tag absent EVERYWHERE → the occupancy (all `digest: null`) — free to push.
+ *   - tag present on ANY endpoint at ANY digest → THROWS, one line per cluster
+ *     (`ip → sha256:… | absent`). A single-origin pre-check must never be
+ *     treated as authoritative anywhere in this path — the exact check that
+ *     produced the v10 incident.
+ *
+ * Fail-open is deliberate on TRANSPORT errors per endpoint? NO — a cluster we
+ * cannot interrogate is a cluster we cannot vouch for, so a non-404/200 answer
+ * is fatal too. Only the not-a-mesh case (above) returns `null`.
+ */
+async function assertTagFreeOnMesh(opts) {
+    const log = opts.log ?? (() => undefined);
+    const host = registryHostOf(opts.registry.server);
+    let endpoints;
+    try {
+        endpoints = await resolveClusterEndpoints(host, opts.resolve4);
+    }
+    catch (err) {
+        log(`  pre-check: ${host} did not resolve (${err.message}); single-target, nothing to pre-check`);
+        return null;
+    }
+    if (endpoints.length <= 1) {
+        log(`  pre-check: ${host} resolves to ${endpoints.length} endpoint(s); single-target, nothing to pre-check`);
+        return null;
+    }
+    const transport = opts.transport ?? defaultTransport(host);
+    const client = makeClient(transport, opts.registry.username, opts.registry.password);
+    const scope = `repository:${opts.registry.repository}:pull,push`;
+    const manifestPath = `/v2/${opts.registry.repository}/manifests/${opts.tag}`;
+    const perEndpoint = await Promise.all(endpoints.map(async (ip) => {
+        const res = await client.send({ ip, method: 'HEAD', path: manifestPath, headers: { Accept: MANIFEST_ACCEPT } }, scope);
+        if (res.status === 200) {
+            const served = header(res, 'docker-content-digest');
+            if (served)
+                return { ip, digest: served };
+            // 200 with no digest: the registry is broken, not the tag's absence.
+            throw new Error(`pre-check: ${ip} served HTTP 200 for ${opts.registry.repository}:${opts.tag} with no Docker-Content-Digest — cannot vouch for the mesh`);
+        }
+        if (res.status === 404)
+            return { ip, digest: null };
+        throw new Error(`pre-check: ${ip} answered HTTP ${res.status} for ${opts.registry.repository}:${opts.tag} — cannot vouch for the mesh`);
+    }));
+    const occupied = perEndpoint.filter((e) => e.digest !== null);
+    if (occupied.length > 0) {
+        const lines = perEndpoint.map((e) => `    ${e.ip} → ${e.digest ?? 'absent'}`);
+        throw new Error(`tag ${opts.tag} is already occupied on the registry mesh (${host}) — a fresh build ` +
+            'cannot take it over: per-cluster immutability would 412 mid-fan-out and can leave ' +
+            'the tag diverged across clusters permanently (#2766). Pick a new tag. Mesh state:\n' +
+            lines.join('\n'));
+    }
+    log(`  pre-check: ${opts.tag} is free on all ${endpoints.length} cluster registries`);
+    return { host, tag: opts.tag, perEndpoint };
+}
 /**
  * The deploy-time entrypoint: take an image just pushed to the federated host
  * (landed on ONE cluster) and guarantee it is present on EVERY cluster Harbor.
@@ -513,7 +747,7 @@ async function resolveAttestationDigest(opts) {
  */
 async function fanOutPushedImage(opts) {
     const log = opts.log ?? (() => undefined);
-    const host = opts.registry.server.replace(/^https?:\/\//, '').replace(/:\d+$/, '');
+    const host = registryHostOf(opts.registry.server);
     let endpoints;
     try {
         endpoints = await resolveClusterEndpoints(host, opts.resolve4);
@@ -542,4 +776,300 @@ async function fanOutPushedImage(opts) {
     }
     return report;
 }
+/**
+ * Is `candidate` a sub-domain of `reference`'s PARENT zone? `harbor.a.b.c` →
+ * parent `a.b.c`, so `harbor-sn1.a.b.c` passes and `harbor-sn1.evil.example`
+ * does not. Matches on the dot boundary, so a look-alike suffix
+ * (`evil-a.b.c` against parent `a.b.c`) is rejected. A single-label reference
+ * has no parent zone and admits nothing.
+ */
+function isSameDnsZone(candidate, reference) {
+    const labels = reference.toLowerCase().split('.');
+    if (labels.length < 2)
+        return false;
+    const parent = labels.slice(1).join('.');
+    return candidate.toLowerCase().endsWith(`.${parent}`);
+}
+/**
+ * Resolve the SINGLE-ORIGIN hostname a `docker push` must be aimed at, by
+ * asking the mesh which name it wants upload traffic on. Returns `null` when
+ * there is nothing to pin (single-target registry, no server-side pin, or a pin
+ * that fails validation) — the caller then pushes at the round-robin name, i.e.
+ * exactly today's behaviour.
+ *
+ * ## Why the push must be pinned at all (#1679)
+ *
+ * `harbor.<domain>` is public-DNS round-robin over three INDEPENDENT Harbors.
+ * A chunked blob upload (`POST` init → `PATCH` → `PUT` finalize) is stateful:
+ * the `_state` token is HMAC'd by the serving registry and the uploaded bytes
+ * sit on that cluster's own ReadWriteOnce PVC. When the client reconnects
+ * between requests and round-robin lands the next hop on a different cluster,
+ * the upload dies (`blob upload invalid` / `invalid secret`). Sharing
+ * `REGISTRY_HTTP_SECRET` and the token CA across the mesh (PR #2763) fixes the
+ * HMAC and the bearer token, but NOT the blob bytes — those are still local to
+ * one cluster. The push has to stay on one origin end to end; the fan-out then
+ * distributes the finished artifact ({@link fanOutPushedImage}).
+ *
+ * ## Why we ASK instead of deriving the name
+ *
+ * Each cluster's registry is already pinned server-side (`REGISTRY_HTTP_HOST`)
+ * and advertises that name in the `Location` of an upload-init. The server
+ * honours it; docker 29.5.2 does not — it keeps the host it DIALLED and reuses
+ * only the path+query of the `Location`, so the server-side pin never reaches
+ * the wire. Reading the advertised host and dialling it ourselves is therefore
+ * the client-side half of a pin the server already publishes. Deriving
+ * `harbor-sn1` from `harbor` by string surgery would instead bake a testnet DNS
+ * convention into a customer-facing CLI, and silently mis-aim any other mesh.
+ *
+ * ## Validation
+ *
+ * The advertised value is server-controlled, and the CLI hands it to
+ * `docker login` / `docker push` — so it gates the robot credential and the
+ * customer's image. Three guards, all fail-CLOSED (return `null`, never throw;
+ * the caller then pushes unpinned, exactly as it would have anyway):
+ *
+ *   1. Not an IP literal — no cert covers one here, and "does this address
+ *      resolve to itself" is not an identity check.
+ *   2. Inside `host`'s own parent DNS zone. A registry may only nominate a name
+ *      the OPERATOR already controls, never an arbitrary internet name.
+ *   3. Resolves to exactly ONE address, and that address is the very cluster
+ *      that advertised it.
+ *
+ * RESIDUAL RISK (accepted, documented): validation and use are two different
+ * DNS lookups — ours via `dns.resolve4`, then docker's own, moments later. A
+ * DNS rebind between them is not detectable from here, because docker resolves
+ * the hostname itself and takes no IP pin. Guard 2 is what makes that
+ * uninteresting: a rebind would need control of the operator's own zone, and an
+ * attacker holding that already owns `harbor.<domain>` itself. Closing the
+ * window completely means not shelling out to `docker` for the push at all
+ * (doing the upload over the Registry-v2 transport in this file, which does pin
+ * IP + SNI) — a much larger change, tracked separately.
+ *
+ * The probe opens one blob-upload session per attempt and cancels it
+ * (`DELETE`), so it leaves no half-open upload behind.
+ */
+async function resolvePushOrigin(opts) {
+    const log = opts.log ?? (() => undefined);
+    const resolve4 = opts.resolve4 ?? ((h) => dns_1.promises.resolve4(h));
+    let endpoints;
+    if (opts.endpoints) {
+        endpoints = opts.endpoints;
+    }
+    else {
+        try {
+            endpoints = await resolveClusterEndpoints(opts.host, resolve4);
+        }
+        catch (err) {
+            log(`  push-pin: ${opts.host} did not resolve (${err.message}); pushing unpinned`);
+            return null;
+        }
+    }
+    if (endpoints.length <= 1) {
+        // A single-target registry (dev / local / a mesh behind one VIP) cannot
+        // split a push across clusters, so there is nothing to pin.
+        return null;
+    }
+    const transport = opts.transport ?? defaultTransport(opts.host);
+    const client = makeClient(transport, opts.username, opts.password);
+    const pushScope = `repository:${opts.repository}:pull,push`;
+    const uploadInitPath = `/v2/${opts.repository}/blobs/uploads/`;
+    for (const ip of endpoints) {
+        // BACKSTOP — this function's contract is "never rejects: return null and
+        // let the caller push unpinned", and NEITHER caller wraps it in try/catch.
+        // The targeted catches inside give better messages; this is what makes the
+        // contract hold structurally rather than by enumeration, including for
+        // edits made later. `continue`/`return` still work normally from inside.
+        try {
+            // Unreachable, or a non-2xx: this cluster tells us nothing, try the next.
+            // A throw here is handled by the BACKSTOP above — deliberately NOT caught
+            // locally as well, so the backstop is load-bearing rather than decorative
+            // (a second handler here made it impossible for any test to notice its
+            // removal).
+            const res = await client.send({ ip, method: 'POST', path: uploadInitPath, headers: { 'Content-Length': '0' } }, pushScope);
+            if (res.status < 200 || res.status >= 300)
+                continue;
+            const location = header(res, 'location');
+            if (!location)
+                continue;
+            // Parse the `Location` ONCE, defensively. It is server-controlled, so it may
+            // be absent, relative, or simply unparseable (`https://`, embedded
+            // whitespace, a control character). Everything below reads this parse — no
+            // second `new URL` and no prefix-shape check standing in for a real parse,
+            // which is how an "unparseable but http-prefixed" value used to throw out of
+            // this function while a request argument was being BUILT, past the trailing
+            // `.catch`, past both callers, turning a graceful fall-back into a hard
+            // deploy abort.
+            // A throw here (`https://`, embedded whitespace, a control character —
+            // all server-controlled) is caught by the BACKSTOP, which moves to the
+            // next cluster and ultimately returns null. Not caught locally as well:
+            // a second handler would be unkillable by any test and could rot.
+            const parsed = /^https?:\/\//i.test(location) ? new URL(location) : null;
+            // Cancel the probe session — best effort, never fatal, but LOGGED: the
+            // "leaves no half-open upload behind" promise must not fail invisibly. Only
+            // a request-target we actually derived is sent; nothing is spliced verbatim.
+            const cancelPath = parsed
+                ? parsed.pathname + parsed.search
+                : location.startsWith('/')
+                    ? location
+                    : null;
+            if (cancelPath) {
+                await client
+                    .send({ ip, method: 'DELETE', path: cancelPath }, pushScope)
+                    .catch((err) => log(`  push-pin: could not cancel the probe upload session on ${ip} ` +
+                    `(${err?.message ?? err}) — it will expire on its own`));
+            }
+            // A relative (or unparseable) `Location` means there is no absolute host to
+            // adopt — this registry advertises no server-side pin.
+            if (!parsed)
+                continue;
+            const advertised = parsed.host; // host[:port] — what docker will be given
+            const hostname = parsed.hostname; // no port — what DNS resolves
+            // Compare on the HOSTNAME: `parsed.host` may carry a port while `opts.host`
+            // never does, so `harbor.<domain>:8443` would otherwise read as a distinct
+            // origin and aim the push straight back at the round-robin name. (Not
+            // separately testable — the round-robin name always has ≥2 A-records, so
+            // guard 3 below would refuse it anyway. Correct here regardless of that.)
+            if (!hostname || hostname.toLowerCase() === opts.host.toLowerCase()) {
+                // The registry echoes the round-robin name → no pin configured.
+                continue;
+            }
+            if (net.isIP(hostname) !== 0) {
+                // A bare IP can carry no valid cert for this mesh (the Harbor certs are
+                // keyed on names), and "does this address resolve back to itself" is not
+                // a meaningful identity check. Rejected explicitly rather than left to
+                // whatever a resolver does with an IP-shaped name.
+                log(`  push-pin: ignoring ${advertised} — an IP literal is not a usable push origin`);
+                continue;
+            }
+            // GUARD 1 — the nomination must stay inside the operator's OWN DNS zone.
+            // Without this a compromised cluster could nominate any name on the
+            // internet, point it at its own IP just long enough to pass guard 2, then
+            // re-point it (short TTL) before docker resolves — sending the robot
+            // credential and the customer image to attacker infrastructure. See the
+            // residual-risk note in the docblock.
+            if (!isSameDnsZone(hostname, opts.host)) {
+                log(`  push-pin: REFUSING ${advertised} — outside ${opts.host}'s own DNS zone; ` +
+                    'a registry may only nominate a name the operator already controls');
+                continue;
+            }
+            // GUARD 2 — the advertised name MUST resolve to exactly the one cluster
+            // that advertised it. A multi-answer name is not a single origin: docker
+            // re-resolves independently and could dial a different address in the same
+            // answer set, reinstating the very cross-cluster hop this pin exists to stop.
+            let advertisedIps;
+            try {
+                advertisedIps = await resolve4(hostname);
+            }
+            catch (err) {
+                log(`  push-pin: ignoring ${advertised} — it does not resolve (${err.message})`);
+                continue;
+            }
+            if (advertisedIps.length !== 1 || advertisedIps[0] !== ip) {
+                log(`  push-pin: REFUSING ${advertised} — it resolves to ${advertisedIps.join(', ') || 'nothing'}, ` +
+                    `not to exactly ${ip} (the cluster that advertised it)`);
+                continue;
+            }
+            log(`  push-pin: ${opts.host} → ${advertised} (single origin ${ip}) for this push`);
+            return advertised;
+        }
+        catch (err) {
+            log(`  push-pin: probe of ${ip} failed (${err?.message ?? String(err)})`);
+        }
+    }
+    log(`  push-pin: no cluster behind ${opts.host} advertised a usable single-origin host; ` +
+        'pushing unpinned (a mid-upload cluster hop may fail — #1679)');
+    return null;
+}
+/**
+ * The pin every push goes through (#2887). Wraps {@link resolvePushOrigin} and
+ * turns its fail-OPEN `null` into a hard failure **when the registry name is
+ * federated** — i.e. when it resolves to more than one origin.
+ *
+ * ## Why the fail-open had to go
+ *
+ * `resolvePushOrigin` returning `null` means "I could not pin this push". Its
+ * callers then pushed at the round-robin name, which is documented as
+ * "exactly today's behaviour" — but on a federated mesh today's behaviour does
+ * not work. Measured on testnet 2026-08-27 against
+ * `harbor.testnet.hsuite.network` (three A records, three INDEPENDENT Harbors,
+ * each signing its own bearer tokens and holding blob-upload sessions on its
+ * own ReadWriteOnce PVC): `Login Succeeded`, then some layers reporting
+ * `Layer already exists` while others sat on `Waiting`, then
+ * `error from registry: unauthorized to access repository: …, action: push`
+ * — on all 6 retries. The identical push aimed at `harbor-sn2.…` succeeded on
+ * the first attempt, then again at sn1 and sn3.
+ *
+ * So the unpinned fall-back is not a degraded push, it is a push that cannot
+ * succeed, and taking it silently costs a full image build plus six retries
+ * before failing with an error message that blames permissions. Refusing up
+ * front — naming the mesh, its origins, and the server-side knob that fixes it
+ * — is strictly better information for the same outcome.
+ *
+ * ## What is still a legitimate no-pin (returns `null`, no throw)
+ *
+ *   - ONE origin. A dev / local / single-VIP registry cannot split a push
+ *     across clusters, so there is nothing to pin and nothing to fail.
+ *   - DNS that cannot enumerate the name at all. `dns.resolve4` (this module's
+ *     resolver) ignores `/etc/hosts` while docker's own resolution does not, so
+ *     an unresolvable name here can still be a perfectly working single-origin
+ *     registry on the developer's machine. Unproven federation is not proven
+ *     federation; we log it and let the push proceed.
+ *
+ * RESIDUAL RISK (accepted, documented): the A-record count is the ONLY signal
+ * we have for "is this mesh federated". A resolver that answers with fewer
+ * records than the mesh really has — a spoofed or on-path answer, or simply a
+ * health-pruned view at query time — makes a three-Harbor mesh look
+ * single-origin, and the push falls back to unpinned. That is not a
+ * credential-disclosure risk (the three validation guards in
+ * {@link resolvePushOrigin} are untouched, and the fall-back still targets the
+ * operator's own round-robin name, resolved independently by docker); the cost
+ * is reinstating the failure mode above — a wasted build and six confusing
+ * retries. Removing it needs a source of mesh cardinality that is not DNS.
+ *
+ * Callers that genuinely want no pin have `--skip-push` (nothing is uploaded,
+ * so there is no upload session to keep on one origin).
+ */
+async function requirePushOrigin(opts) {
+    const log = opts.log ?? (() => undefined);
+    const resolve4 = opts.resolve4 ?? ((h) => dns_1.promises.resolve4(h));
+    // ONE DNS answer decides both "is this federated?" and "which origins do we
+    // probe?" — resolving twice lets the two disagree, and a narrowing answer
+    // between them is exactly how a mesh could look single-origin to one check
+    // and federated to the other.
+    let endpoints;
+    try {
+        endpoints = opts.endpoints ?? (await resolveClusterEndpoints(opts.host, resolve4));
+    }
+    catch (err) {
+        // RETURN, do not delegate. Handing the un-enumerated `opts` to
+        // `resolvePushOrigin` would let IT run a second, independent lookup — and a
+        // resolver that fails once then answers with the full federated set would
+        // leave the decision below reading a stale "not federated" while the nested
+        // call had just proved the opposite. That is the exact disagreement the
+        // single-answer rule above exists to prevent, so the failure path must
+        // honour it too. Nothing is lost: the probe needs the IP list, so a
+        // delegated call could only have re-derived what we just failed to get.
+        log(`  push-pin: ${opts.host} did not resolve via DNS (${err.message}); ` +
+            'cannot tell whether this registry is federated — pushing unpinned');
+        return null;
+    }
+    const origin = await resolvePushOrigin({ ...opts, endpoints });
+    if (origin)
+        return origin;
+    if (endpoints.length > 1) {
+        throw new Error(`cannot pin the docker push for ${opts.repository}: ${opts.host} resolves to ` +
+            `${endpoints.length} independent registries (${endpoints.join(', ')}) and none of them ` +
+            'advertised a usable single-origin host to push at.\n' +
+            '  An UNPINNED push across a federated Harbor mesh cannot succeed (#1679): every ' +
+            'request re-resolves the round-robin name, each cluster signs its own bearer tokens, ' +
+            'and the uploaded blob bytes live on the originating cluster\'s own ReadWriteOnce PVC — ' +
+            'so the layers scatter and the push dies mid-upload (`blob upload invalid`, or ' +
+            '`unauthorized to access repository: …, action: push`) on every retry.\n' +
+            '  Refusing to start a push that cannot finish. Fix the mesh: each cluster\'s registry ' +
+            'must advertise its OWN single-A name via REGISTRY_HTTP_HOST (k8s/harbor/values-snN.yaml ' +
+            '→ registry.registry.extraEnvVars; asserted by charts/smart-engines/tests/helm-template.bats). ' +
+            'Or re-run with --skip-push if the image is already on the mesh.');
+    }
+    return null;
+}
 //# sourceMappingURL=registry-fanout.js.map