@specific.dev/spectest 0.131.8 → 0.131.10

package/dist/components/k3s-join.d.ts

@@ -0,0 +1,2 @@
+export declare const JOIN_DIRECTORY = "/etc/spectest-k3s-join";
+export declare function prepareK3sJoin(server: boolean, directory?: string): string;

package/dist/components/k3s-join.js

@@ -0,0 +1,13 @@
+export const JOIN_DIRECTORY = "/etc/spectest-k3s-join";
+// The join volume is private to one cluster and is captured with its network.
+// Use the server's numeric address from the beginning: K3s otherwise demotes
+// the DNS fallback after discovery and closes in-flight bootstrap requests.
+export function prepareK3sJoin(server, directory = JOIN_DIRECTORY) {
+    if (server)
+        return ('SERVER="$(ip -4 route get 1.1.1.1 | awk \'{for (i=1;i<NF;i++) if ($i=="src") {print $(i+1); exit}}\')"; ' +
+            'case "$SERVER" in ""|*[!0-9.]*) echo "spectest: missing server IPv4 address" >&2; exit 1;; esac; ' +
+            `printf '%s\\n' "$SERVER" > ${directory}/server.tmp && mv ${directory}/server.tmp ${directory}/server || exit 1; ` +
+            `if [ ! -s ${directory}/token ]; then (umask 077; cat /proc/sys/kernel/random/uuid > ${directory}/token) || exit 1; fi; `);
+    return (`while [ ! -s ${directory}/token ] || [ ! -s ${directory}/server ]; do sleep 0.1; done; ` +
+        `SERVER="$(cat ${directory}/server)"; `);
+}

package/dist/components/k3s.js

@@ -1,5 +1,6 @@
 import { isIP } from "node:net";
 import { waitForK3sNodes } from "./k3s-nodes.js";
+import { JOIN_DIRECTORY, prepareK3sJoin } from "./k3s-join.js";
 import { installPodTrust } from "./k3s-trust.js";
 import { k3sCoverage } from "./k3s-coverage.js";
 import { waitForAll } from "../harness/concurrency.js";
@@ -21,7 +22,6 @@ import { deepUnwrap, readTag, wrap } from "../inspect.js";
 import { recorderAnnotate, recorderRemove } from "../recorder.js";
 /** Port the in-cluster registry listens on (plain HTTP). */
 const K3S_REGISTRY_PORT = 5000;
-const JOIN_DIRECTORY = "/etc/spectest-k3s-join";
 /** The cluster's own containerd root, inside the k3s container. */
 const CLUSTER_STORE_TARGET = "/var/lib/rancher/k3s/agent/containerd";
 /** The socket of the throwaway daemon that prepares the store. */
@@ -1374,7 +1374,7 @@ function createK3sCluster(opts, count) {
             "--kubelet-arg=kubelet-cgroups=/spectest-$(hostname)/k3s",
             "--kubelet-arg=runtime-cgroups=/spectest-$(hostname)/k3s",
             `--token-file=${JOIN_DIRECTORY}/token`,
-            ...(server ? [] : [`--server=https://${cluster}.internal:6443`]),
+            ...(server ? [] : ['--server="https://$SERVER:6443"']),
             // CoreDNS / pod DNS upstream. Without this, k3s sees only the
             // loopback 127.0.0.11 (Docker's embedded DNS) in the container's
             // /etc/resolv.conf, decides no usable nameserver exists, and writes a
@@ -1464,9 +1464,7 @@ function createK3sCluster(opts, count) {
             'sed "s/[^ ][^ ]*/+&/g" "/sys/fs/cgroup/$CG/cgroup.controllers" > "/sys/fs/cgroup/$CG/cgroup.subtree_control" || exit 1; ' +
             'else for C in /sys/fs/cgroup/*; do [ ! -d "$C" ] || mkdir -p "$C/$CG/k3s" || exit 1; done; fi; ' +
             storePrepareSh() +
-            (server
-                ? `if [ ! -s ${JOIN_DIRECTORY}/token ]; then (umask 077; cat /proc/sys/kernel/random/uuid > ${JOIN_DIRECTORY}/token); fi; `
-                : `while [ ! -s ${JOIN_DIRECTORY}/token ]; do sleep 0.1; done; `) +
+            prepareK3sJoin(server) +
             // Sticky and environment-local stores use overlayfs.
             `exec ${nodeArgs} --snapshotter=$SNAP`;
         // Plain /readyz probe. On a warm zot cache the cluster's images are

package/dist/components/s3-runtime/README.md

@@ -27,6 +27,12 @@ one backend snapshot, including closed byte ranges, exact ETag conditions,
 version selection and SHA256 part receipts. Date-based copy conditions are
 explicitly unsupported (501), rather than silently ignored.
 
+Multipart completion stores its opaque ETag on the same in-memory object version
+as the bytes. HEAD/GET, listings, copy conditions and atomic write conditions use
+that tag; ordinary PUT and CopyObject retain their own payload-derived tags.
+Versioned HEAD reads the requested version. The optional backend write port keeps
+completion tags out of client metadata and avoids a separate post-write update.
+
 Run `./test.sh` from this directory to exercise real HTTP requests against the
 same patched sources used in the image, including corrupt retries, conditional
 completion races, abort, metadata reconciliation and unchecksummed uploads.

package/dist/components/s3-runtime/main_test.go

@@ -147,39 +147,54 @@ func TestMultipartSHA256IntegrityAndReconciliation(t *testing.T) {
 }
 
 func TestMultipartConditionalCompletionIsAtomic(t *testing.T) {
-	p := newProbe(t)
-	paths := []string{p.start("same", true), p.start("same", true)}
-	parts := []string{p.part(paths[0], 1, []byte("first"), true), p.part(paths[1], 1, []byte("second"), true)}
-	statuses := make(chan int, 2)
-	var wg sync.WaitGroup
-	for i := range paths {
-		wg.Add(1)
-		go func(i int) {
-			defer wg.Done()
-			req, _ := http.NewRequest("POST", p.server.URL+paths[i], bytes.NewReader(completion(parts[i])))
-			req.Header.Set("If-None-Match", "*")
-			res, err := p.server.Client().Do(req)
-			if err != nil {
-				statuses <- 0
-				return
+	for _, replace := range []bool{false, true} {
+		t.Run(fmt.Sprint(replace), func(t *testing.T) {
+			p := newProbe(t)
+			header, tag := "If-None-Match", "*"
+			if replace {
+				seed := p.start("same", true)
+				part := p.part(seed, 1, []byte("original"), true)
+				response := p.request("POST", seed, completion(part), nil, 200)
+				var receipt struct{ ETag string }
+				if err := xml.Unmarshal(response.body, &receipt); err != nil {
+					t.Fatal(err)
+				}
+				header, tag = "If-Match", receipt.ETag
 			}
-			defer res.Body.Close()
-			io.Copy(io.Discard, res.Body)
-			statuses <- res.StatusCode
-		}(i)
-	}
-	wg.Wait()
-	close(statuses)
-	counts := map[int]int{}
-	for s := range statuses {
-		counts[s]++
-	}
-	if counts[200] != 1 || counts[412] != 1 {
-		t.Fatalf("competing creates: %v", counts)
-	}
-	body := p.request("GET", "/uploads/same", nil, nil, 200).body
-	if string(body) != "first" && string(body) != "second" {
-		t.Fatalf("bad winner: %s", body)
+			paths := []string{p.start("same", true), p.start("same", true)}
+			parts := []string{p.part(paths[0], 1, []byte("first"), true), p.part(paths[1], 1, []byte("second"), true)}
+			statuses := make(chan int, 2)
+			var wg sync.WaitGroup
+			for i := range paths {
+				wg.Add(1)
+				go func(i int) {
+					defer wg.Done()
+					req, _ := http.NewRequest("POST", p.server.URL+paths[i], bytes.NewReader(completion(parts[i])))
+					req.Header.Set(header, tag)
+					res, err := p.server.Client().Do(req)
+					if err != nil {
+						statuses <- 0
+						return
+					}
+					defer res.Body.Close()
+					io.Copy(io.Discard, res.Body)
+					statuses <- res.StatusCode
+				}(i)
+			}
+			wg.Wait()
+			close(statuses)
+			counts := map[int]int{}
+			for s := range statuses {
+				counts[s]++
+			}
+			if counts[200] != 1 || counts[412] != 1 {
+				t.Fatalf("competing creates: %v", counts)
+			}
+			body := p.request("GET", "/uploads/same", nil, nil, 200).body
+			if string(body) != "first" && string(body) != "second" {
+				t.Fatalf("bad winner: %s", body)
+			}
+		})
 	}
 }
 
@@ -280,3 +295,78 @@ func TestUploadPartCopyVersionChecksumAndAbort(t *testing.T) {
 	p.request("PUT", path+"&partNumber=1", nil, headers, 404)
 	p.request("DELETE", path, nil, nil, 204)
 }
+
+func TestMultipartSourceEntityTagSurvivesCopyVersionsAndOverwrite(t *testing.T) {
+	for _, count := range []int{1, 2} {
+		t.Run(fmt.Sprint(count), func(t *testing.T) {
+			p := newProbe(t)
+			p.request("PUT", "/uploads?versioning", []byte("<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"), nil, 200)
+			body := bytes.Repeat([]byte("a"), 6*1024*1024)
+			path := p.start("staged", false)
+			parts := []string{}
+			if count == 1 {
+				parts = append(parts, p.part(path, 1, body, false))
+			} else {
+				parts = append(parts, p.part(path, 1, body[:5*1024*1024], false), p.part(path, 2, body[5*1024*1024:], false))
+			}
+			complete := p.request("POST", path, completion(parts...), nil, 200)
+			var receipt struct{ ETag string }
+			if err := xml.Unmarshal(complete.body, &receipt); err != nil || !strings.Contains(receipt.ETag, fmt.Sprintf("-%d", count)) {
+				t.Fatalf("completion tag: %s: %v", complete.body, err)
+			}
+			version := complete.header.Get("x-amz-version-id")
+			if version == "" {
+				t.Fatal("missing completed version")
+			}
+			destination := p.start("copied-staging", false)
+			headers := map[string]string{"x-amz-copy-source": "/uploads/staged", "x-amz-copy-source-if-match": receipt.ETag, "x-amz-copy-source-range": fmt.Sprintf("bytes=0-%d", len(body)-1)}
+			copied := p.request("PUT", destination+"&partNumber=1", nil, headers, 200)
+			var part struct{ ETag string }
+			if err := xml.Unmarshal(copied.body, &part); err != nil {
+				t.Fatal(err)
+			}
+			for _, method := range []string{"HEAD", "GET"} {
+				got := p.request(method, "/uploads/staged", nil, nil, 200)
+				if got.header.Get("ETag") != receipt.ETag {
+					t.Fatalf("%s lost completion tag: %s", method, got.header.Get("ETag"))
+				}
+				p.request(method, "/uploads/staged", nil, map[string]string{"If-None-Match": receipt.ETag}, 304)
+			}
+			for _, query := range []string{"?list-type=2", "?versions"} {
+				got := p.request("GET", "/uploads"+query, nil, nil, 200)
+				var list struct {
+					Contents []struct{ ETag string }
+					Version  []struct{ ETag string }
+				}
+				if err := xml.Unmarshal(got.body, &list); err != nil {
+					t.Fatal(err)
+				}
+				tags := []string{}
+				for _, row := range list.Contents {
+					tags = append(tags, row.ETag)
+				}
+				for _, row := range list.Version {
+					tags = append(tags, row.ETag)
+				}
+				if len(tags) != 1 || tags[0] != receipt.ETag {
+					t.Fatalf("list lost tag: %v", tags)
+				}
+			}
+			put := p.request("PUT", "/uploads/staged", []byte("replacement"), map[string]string{"If-Match": receipt.ETag}, 200)
+			if put.header.Get("ETag") == receipt.ETag {
+				t.Fatal("normal overwrite inherited multipart tag")
+			}
+			p.request("PUT", destination+"&partNumber=1", nil, headers, 412)
+			headers["x-amz-copy-source"] += "?versionId=" + version
+			p.request("PUT", destination+"&partNumber=1", nil, headers, 200)
+			head := p.request("HEAD", "/uploads/staged?versionId="+version, nil, nil, 200)
+			if head.header.Get("ETag") != receipt.ETag {
+				t.Fatal("version lost its tag")
+			}
+			p.request("POST", destination, completion(fmt.Sprintf("<Part><PartNumber>1</PartNumber><ETag>%s</ETag></Part>", part.ETag)), nil, 200)
+			if got := p.request("GET", "/uploads/copied-staging", nil, nil, 200); !bytes.Equal(got.body, body) {
+				t.Fatal("copied multipart staging bytes changed")
+			}
+		})
+	}
+}

package/dist/components/s3-runtime/multipart.patch

@@ -350,3 +350,199 @@
 +		ChecksumSHA256 string      `xml:"ChecksumSHA256"`
 +	}{ETag: etag, LastModified: NewContentTime(g.timeSource.Now()), ChecksumSHA256: base64.StdEncoding.EncodeToString(digest.Sum(nil))})
 +}
+--- a/vendor/github.com/johannesboyne/gofakes3/backend.go
++++ b/vendor/github.com/johannesboyne/gofakes3/backend.go
+@@ -23,7 +23,9 @@
+ 	Size     int64
+ 	Contents io.ReadCloser
+ 	Hash     []byte
+-	Range    *ObjectRange
++	// EntityTag is the stored opaque tag; Hash remains the payload MD5.
++	EntityTag string
++	Range     *ObjectRange
+ 
+ 	// VersionID will be empty if bucket versioning has not been enabled.
+ 	VersionID VersionID
+@@ -31,6 +33,21 @@
+ 	// If versioning is enabled for the bucket, this is true if this object version
+ 	// is a delete marker.
+ 	IsDeleteMarker bool
++}
++
++// ETag returns the version-owned tag, retaining compatibility with backends
++// that only supply payload hashes.
++func (object *Object) ETag() string {
++	if object.EntityTag != "" {
++		return object.EntityTag
++	}
++	return FormatETag(object.Hash)
++}
++
++// MultipartObjectBackend atomically stores completion bytes, metadata and ETag
++// under the same conditional-write lock. The tag never enters client metadata.
++type MultipartObjectBackend interface {
++	PutMultipartObject(bucket, key string, metadata map[string]string, input io.Reader, size int64, tag string, conditions *PutConditions) (PutObjectResult, error)
+ }
+ 
+ type ObjectList struct {
+@@ -150,7 +167,8 @@
+ 
+ 	// Hash is the MD5 hash of the object content (used for ETag comparison)
+ 	// Only required if Exists is true
+-	Hash []byte
++	Hash      []byte
++	EntityTag string
+ }
+ 
+ // ObjectInfo is a deprecated alias for ConditionalObjectInfo.
+@@ -182,7 +200,10 @@
+ 		if len(expectedETag) >= 2 && expectedETag[0] == '"' && expectedETag[len(expectedETag)-1] == '"' {
+ 			expectedETag = expectedETag[1 : len(expectedETag)-1]
+ 		}
+-		actualETag := hex.EncodeToString(objectInfo.Hash)
++		actualETag := strings.Trim(objectInfo.EntityTag, "\"")
++		if actualETag == "" {
++			actualETag = hex.EncodeToString(objectInfo.Hash)
++		}
+ 		if expectedETag != actualETag {
+ 			return ErrorMessage(ErrPreconditionFailed, "The ETag does not match")
+ 		}
+--- a/vendor/github.com/johannesboyne/gofakes3/uploader.go
++++ b/vendor/github.com/johannesboyne/gofakes3/uploader.go
+@@ -496,7 +496,12 @@
+ 		input.ChecksumSHA256 = ""
+ 		input.ChecksumType = ""
+ 	}
+-	result, err := u.storage.PutObject(bucket, object, mpu.Meta, bytes.NewReader(body), int64(len(body)), input.Conditions)
++	var result PutObjectResult
++	if backend, ok := u.storage.(MultipartObjectBackend); ok {
++		result, err = backend.PutMultipartObject(bucket, object, mpu.Meta, bytes.NewReader(body), int64(len(body)), etag, input.Conditions)
++	} else {
++		result, err = u.storage.PutObject(bucket, object, mpu.Meta, bytes.NewReader(body), int64(len(body)), input.Conditions)
++	}
+ 	if err != nil {
+ 		return "", "", err
+ 	}
+--- a/vendor/github.com/johannesboyne/gofakes3/gofakes3.go
++++ b/vendor/github.com/johannesboyne/gofakes3/gofakes3.go
+@@ -535,7 +535,7 @@
+ 		w.Header().Set("x-amz-version-id", string(obj.VersionID))
+ 	}
+ 
+-	etag := `"` + hex.EncodeToString(obj.Hash) + `"`
++	etag := obj.ETag()
+ 	w.Header().Set("ETag", etag)
+ 
+ 	if r.Header.Get("If-None-Match") == etag {
+@@ -576,7 +576,16 @@
+ 		return err
+ 	}
+ 
+-	obj, err := g.storage.HeadObject(bucket, object)
++	var obj *Object
++	var err error
++	if versionID == "" {
++		obj, err = g.storage.HeadObject(bucket, object)
++	} else {
++		if g.versioned == nil {
++			return ErrNotImplemented
++		}
++		obj, err = g.versioned.HeadObjectVersion(bucket, object, versionID)
++	}
+ 	if err != nil {
+ 		return err
+ 	}
+--- a/vendor/github.com/johannesboyne/gofakes3/upload_part_copy.go
++++ b/vendor/github.com/johannesboyne/gofakes3/upload_part_copy.go
+@@ -56,7 +56,7 @@
+ 	if src.IsDeleteMarker {
+ 		return ErrInvalidArgument
+ 	}
+-	tag := FormatETag(src.Hash)
++	tag := src.ETag()
+ 	if expected := r.Header.Get("x-amz-copy-source-if-match"); expected != "" && expected != tag {
+ 		return ErrPreconditionFailed
+ 	}
+--- a/vendor/github.com/johannesboyne/gofakes3/backend/s3mem/backend.go
++++ b/vendor/github.com/johannesboyne/gofakes3/backend/s3mem/backend.go
+@@ -119,7 +119,7 @@
+ 			response.Add(&gofakes3.Content{
+ 				Key:          item.data.name,
+ 				LastModified: gofakes3.NewContentTime(item.data.lastModified),
+-				ETag:         gofakes3.FormatETag(item.data.hash),
++				ETag:         item.data.entityTag,
+ 				Size:         int64(len(item.data.body)),
+ 			})
+ 		}
+@@ -240,6 +240,16 @@
+ 	size int64,
+ 	conditions *gofakes3.PutConditions,
+ ) (result gofakes3.PutObjectResult, err error) {
++	return db.putObject(bucketName, objectName, meta, input, size, "", conditions)
++}
++
++// PutMultipartObject retains the completion tag in the same object version as
++// its bytes and uses the ordinary atomic conditional-write admission.
++func (db *Backend) PutMultipartObject(bucketName, objectName string, meta map[string]string, input io.Reader, size int64, tag string, conditions *gofakes3.PutConditions) (gofakes3.PutObjectResult, error) {
++	return db.putObject(bucketName, objectName, meta, input, size, tag, conditions)
++}
++
++func (db *Backend) putObject(bucketName, objectName string, meta map[string]string, input io.Reader, size int64, tag string, conditions *gofakes3.PutConditions) (result gofakes3.PutObjectResult, err error) {
+ 	// No need to lock the backend while we read the data into memory; it holds
+ 	// the write lock open unnecessarily, and could be blocked for an unreasonably
+ 	// long time by a connection timing out:
+@@ -272,11 +282,15 @@
+ 	}
+ 
+ 	hash := md5.Sum(bts)
++	if tag == "" {
++		tag = gofakes3.FormatETag(hash[:])
++	}
+ 
+ 	item := &bucketData{
+ 		name:         objectName,
+ 		body:         bts,
+ 		hash:         hash[:],
++		entityTag:    tag,
+ 		metadata:     meta,
+ 		lastModified: db.timeSource.Now(),
+ 	}
+@@ -557,7 +571,7 @@
+ 					IsLatest:     version == object.data,
+ 					LastModified: gofakes3.NewContentTime(version.lastModified),
+ 					Size:         int64(len(version.body)),
+-					ETag:         gofakes3.FormatETag(version.hash),
++					ETag:         version.entityTag,
+ 				}
+ 				if bucket.versioning != gofakes3.VersioningNone { // S300005
+ 					resultVer.VersionID = version.versionID
+@@ -587,8 +601,9 @@
+ 		return &gofakes3.ConditionalObjectInfo{Exists: false}, nil
+ 	}
+ 	return &gofakes3.ConditionalObjectInfo{
+-		Exists: true,
+-		Hash:   existing.data.hash,
++		Exists:    true,
++		Hash:      existing.data.hash,
++		EntityTag: existing.data.entityTag,
+ 	}, nil
+ }
+ 
+--- a/vendor/github.com/johannesboyne/gofakes3/backend/s3mem/bucket.go
++++ b/vendor/github.com/johannesboyne/gofakes3/backend/s3mem/bucket.go
+@@ -118,6 +118,7 @@
+ 	deleteMarker bool
+ 	body         []byte
+ 	hash         []byte
++	entityTag    string
+ 	metadata     map[string]string
+ }
+ 
+@@ -150,6 +151,7 @@
+ 	return &gofakes3.Object{
+ 		Name:           bi.name,
+ 		Hash:           bi.hash,
++		EntityTag:      bi.entityTag,
+ 		Metadata:       bi.metadata,
+ 		Size:           sz,
+ 		Range:          rnge,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@specific.dev/spectest",
-  "version": "0.131.8",
+  "version": "0.131.10",
   "description": "Spectest SDK for defining test environments in TypeScript.",
   "type": "module",
   "main": "./dist/index.js",

package/src/components/k3s-docker.test.ts

@@ -65,6 +65,13 @@ for (const count of [1, 3]) test.skipIf(!process.env.SPECTEST_TEST_K3S_DOCKER)(`
       await Bun.sleep(250);
     }
     await primary.setup!({ name, helpers, exec } as unknown as ServiceSetupContext<K3sHelpers>);
+    // Discovery must retain the bootstrap address. A DNS fallback is demoted
+    // to standby and closes active requests, which can kill initial Flannel
+    // authorization even though the API server remains healthy.
+    for (const node of names.filter(node => node !== name)) {
+      const config = JSON.parse(await checked(["docker", "exec", node, "cat", "/var/lib/rancher/k3s/agent/etc/k3s-agent-load-balancer.json"]));
+      expect(config.ServerAddresses).toContain(new URL(config.ServerURL).host);
+    }
     await helpers.apply(JSON.stringify({ apiVersion: "apps/v1", kind: "DaemonSet", metadata: { name: "web" }, spec: {
       selector: { matchLabels: { app: "web" } }, template: { metadata: { labels: { app: "web" } }, spec: {
         containers: [{ name: "web", image: "busybox:1.37", imagePullPolicy: "Never",

package/src/components/k3s-join.test.ts

@@ -0,0 +1,38 @@
+import { expect, test } from "bun:test";
+import { mkdtemp, readFile, rm, stat } from "node:fs/promises";
+import { networkInterfaces, tmpdir } from "node:os";
+import { join } from "node:path";
+import { prepareK3sJoin } from "./k3s-join";
+
+test("concurrent agents join the server's real numeric identity and retain the private token on restart", async () => {
+  const directory = await mkdtemp(join(tmpdir(), "k3s-join-"));
+  const children: ReturnType<typeof Bun.spawn>[] = [];
+  const start = (server: boolean) => {
+    const child = Bun.spawn(["sh", "-c", prepareK3sJoin(server, directory) + 'printf "%s" "$SERVER"'], {
+      stdout: "pipe", stderr: "pipe",
+    });
+    children.push(child);
+    return child;
+  };
+  const finish = async (child: ReturnType<typeof start>) => {
+    expect(await child.exited).toBe(0);
+    return new Response(child.stdout).text();
+  };
+  try {
+    const agents = [start(false), start(false)];
+    const address = await finish(start(true));
+    const localAddresses = Object.values(networkInterfaces()).flat().filter(a => a?.family === "IPv4").map(a => a!.address);
+    expect(localAddresses).toContain(address);
+    expect(await Promise.all(agents.map(finish))).toEqual([address, address]);
+    expect((await readFile(join(directory, "server"), "utf8")).trim()).toBe(address);
+    const token = await readFile(join(directory, "token"), "utf8");
+    expect(token.trim()).toMatch(/^[0-9a-f-]{36}$/);
+    expect((await stat(join(directory, "token"))).mode & 0o777).toBe(0o600);
+    expect(await finish(start(true))).toBe(address);
+    expect(await readFile(join(directory, "token"), "utf8")).toBe(token);
+  } finally {
+    for (const child of children) if (child.exitCode === null) child.kill();
+    await Promise.all(children.map(child => child.exited));
+    await rm(directory, { recursive: true, force: true });
+  }
+}, 10_000);

package/src/components/k3s-join.ts

@@ -0,0 +1,17 @@
+export const JOIN_DIRECTORY = "/etc/spectest-k3s-join";
+
+// The join volume is private to one cluster and is captured with its network.
+// Use the server's numeric address from the beginning: K3s otherwise demotes
+// the DNS fallback after discovery and closes in-flight bootstrap requests.
+export function prepareK3sJoin(server: boolean, directory = JOIN_DIRECTORY): string {
+  if (server) return (
+    'SERVER="$(ip -4 route get 1.1.1.1 | awk \'{for (i=1;i<NF;i++) if ($i=="src") {print $(i+1); exit}}\')"; ' +
+    'case "$SERVER" in ""|*[!0-9.]*) echo "spectest: missing server IPv4 address" >&2; exit 1;; esac; ' +
+    `printf '%s\\n' "$SERVER" > ${directory}/server.tmp && mv ${directory}/server.tmp ${directory}/server || exit 1; ` +
+    `if [ ! -s ${directory}/token ]; then (umask 077; cat /proc/sys/kernel/random/uuid > ${directory}/token) || exit 1; fi; `
+  );
+  return (
+    `while [ ! -s ${directory}/token ] || [ ! -s ${directory}/server ]; do sleep 0.1; done; ` +
+    `SERVER="$(cat ${directory}/server)"; `
+  );
+}

package/src/components/k3s-nodes.test.ts

@@ -22,7 +22,7 @@ test("one and multiple nodes use the same topology and preserve primary override
   for (const name of ["cluster-node-2", "cluster-node-3"]) {
     const node = three.services[name];
     expect(node.command).toContain("exec k3s agent");
-    expect(node.command).toContain("--server=https://cluster.internal:6443");
+    expect(node.command).toContain('--server="https://$SERVER:6443"');
     expect(node.command).toContain("--kubelet-arg=cgroup-root=/spectest-$(hostname)");
     expect(node.dependsOn).toBeUndefined(); // joins while the server boots
     expect(node.imageBuilds).toHaveLength(0); // build once, import into every node

package/src/components/k3s.ts

@@ -1,5 +1,6 @@
 import { isIP } from "node:net";
 import { waitForK3sNodes } from "./k3s-nodes.js";
+import { JOIN_DIRECTORY, prepareK3sJoin } from "./k3s-join.js";
 import { installPodTrust } from "./k3s-trust.js";
 import { k3sCoverage } from "./k3s-coverage.js";
 import { waitForAll } from "../harness/concurrency.js";
@@ -189,7 +190,6 @@ export interface K3sOptions {
 
 /** Port the in-cluster registry listens on (plain HTTP). */
 const K3S_REGISTRY_PORT = 5000;
-const JOIN_DIRECTORY = "/etc/spectest-k3s-join";
 
 /** The cluster's own containerd root, inside the k3s container. */
 const CLUSTER_STORE_TARGET = "/var/lib/rancher/k3s/agent/containerd";
@@ -2008,7 +2008,7 @@ function createK3sCluster(opts: K3sOptions, count: number) {
       "--kubelet-arg=kubelet-cgroups=/spectest-$(hostname)/k3s",
       "--kubelet-arg=runtime-cgroups=/spectest-$(hostname)/k3s",
       `--token-file=${JOIN_DIRECTORY}/token`,
-      ...(server ? [] : [`--server=https://${cluster}.internal:6443`]),
+      ...(server ? [] : ['--server="https://$SERVER:6443"']),
       // CoreDNS / pod DNS upstream. Without this, k3s sees only the
       // loopback 127.0.0.11 (Docker's embedded DNS) in the container's
       // /etc/resolv.conf, decides no usable nameserver exists, and writes a
@@ -2099,9 +2099,7 @@ function createK3sCluster(opts: K3sOptions, count: number) {
       'sed "s/[^ ][^ ]*/+&/g" "/sys/fs/cgroup/$CG/cgroup.controllers" > "/sys/fs/cgroup/$CG/cgroup.subtree_control" || exit 1; ' +
       'else for C in /sys/fs/cgroup/*; do [ ! -d "$C" ] || mkdir -p "$C/$CG/k3s" || exit 1; done; fi; ' +
       storePrepareSh() +
-      (server
-        ? `if [ ! -s ${JOIN_DIRECTORY}/token ]; then (umask 077; cat /proc/sys/kernel/random/uuid > ${JOIN_DIRECTORY}/token); fi; `
-        : `while [ ! -s ${JOIN_DIRECTORY}/token ]; do sleep 0.1; done; `) +
+      prepareK3sJoin(server) +
       // Sticky and environment-local stores use overlayfs.
       `exec ${nodeArgs} --snapshotter=$SNAP`;
     // Plain /readyz probe. On a warm zot cache the cluster's images are

package/src/components/s3-runtime/README.md

@@ -27,6 +27,12 @@ one backend snapshot, including closed byte ranges, exact ETag conditions,
 version selection and SHA256 part receipts. Date-based copy conditions are
 explicitly unsupported (501), rather than silently ignored.
 
+Multipart completion stores its opaque ETag on the same in-memory object version
+as the bytes. HEAD/GET, listings, copy conditions and atomic write conditions use
+that tag; ordinary PUT and CopyObject retain their own payload-derived tags.
+Versioned HEAD reads the requested version. The optional backend write port keeps
+completion tags out of client metadata and avoids a separate post-write update.
+
 Run `./test.sh` from this directory to exercise real HTTP requests against the
 same patched sources used in the image, including corrupt retries, conditional
 completion races, abort, metadata reconciliation and unchecksummed uploads.

package/src/components/s3-runtime/main_test.go

@@ -147,39 +147,54 @@ func TestMultipartSHA256IntegrityAndReconciliation(t *testing.T) {
 }
 
 func TestMultipartConditionalCompletionIsAtomic(t *testing.T) {
-	p := newProbe(t)
-	paths := []string{p.start("same", true), p.start("same", true)}
-	parts := []string{p.part(paths[0], 1, []byte("first"), true), p.part(paths[1], 1, []byte("second"), true)}
-	statuses := make(chan int, 2)
-	var wg sync.WaitGroup
-	for i := range paths {
-		wg.Add(1)
-		go func(i int) {
-			defer wg.Done()
-			req, _ := http.NewRequest("POST", p.server.URL+paths[i], bytes.NewReader(completion(parts[i])))
-			req.Header.Set("If-None-Match", "*")
-			res, err := p.server.Client().Do(req)
-			if err != nil {
-				statuses <- 0
-				return
+	for _, replace := range []bool{false, true} {
+		t.Run(fmt.Sprint(replace), func(t *testing.T) {
+			p := newProbe(t)
+			header, tag := "If-None-Match", "*"
+			if replace {
+				seed := p.start("same", true)
+				part := p.part(seed, 1, []byte("original"), true)
+				response := p.request("POST", seed, completion(part), nil, 200)
+				var receipt struct{ ETag string }
+				if err := xml.Unmarshal(response.body, &receipt); err != nil {
+					t.Fatal(err)
+				}
+				header, tag = "If-Match", receipt.ETag
 			}
-			defer res.Body.Close()
-			io.Copy(io.Discard, res.Body)
-			statuses <- res.StatusCode
-		}(i)
-	}
-	wg.Wait()
-	close(statuses)
-	counts := map[int]int{}
-	for s := range statuses {
-		counts[s]++
-	}
-	if counts[200] != 1 || counts[412] != 1 {
-		t.Fatalf("competing creates: %v", counts)
-	}
-	body := p.request("GET", "/uploads/same", nil, nil, 200).body
-	if string(body) != "first" && string(body) != "second" {
-		t.Fatalf("bad winner: %s", body)
+			paths := []string{p.start("same", true), p.start("same", true)}
+			parts := []string{p.part(paths[0], 1, []byte("first"), true), p.part(paths[1], 1, []byte("second"), true)}
+			statuses := make(chan int, 2)
+			var wg sync.WaitGroup
+			for i := range paths {
+				wg.Add(1)
+				go func(i int) {
+					defer wg.Done()
+					req, _ := http.NewRequest("POST", p.server.URL+paths[i], bytes.NewReader(completion(parts[i])))
+					req.Header.Set(header, tag)
+					res, err := p.server.Client().Do(req)
+					if err != nil {
+						statuses <- 0
+						return
+					}
+					defer res.Body.Close()
+					io.Copy(io.Discard, res.Body)
+					statuses <- res.StatusCode
+				}(i)
+			}
+			wg.Wait()
+			close(statuses)
+			counts := map[int]int{}
+			for s := range statuses {
+				counts[s]++
+			}
+			if counts[200] != 1 || counts[412] != 1 {
+				t.Fatalf("competing creates: %v", counts)
+			}
+			body := p.request("GET", "/uploads/same", nil, nil, 200).body
+			if string(body) != "first" && string(body) != "second" {
+				t.Fatalf("bad winner: %s", body)
+			}
+		})
 	}
 }
 
@@ -280,3 +295,78 @@ func TestUploadPartCopyVersionChecksumAndAbort(t *testing.T) {
 	p.request("PUT", path+"&partNumber=1", nil, headers, 404)
 	p.request("DELETE", path, nil, nil, 204)
 }
+
+func TestMultipartSourceEntityTagSurvivesCopyVersionsAndOverwrite(t *testing.T) {
+	for _, count := range []int{1, 2} {
+		t.Run(fmt.Sprint(count), func(t *testing.T) {
+			p := newProbe(t)
+			p.request("PUT", "/uploads?versioning", []byte("<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"), nil, 200)
+			body := bytes.Repeat([]byte("a"), 6*1024*1024)
+			path := p.start("staged", false)
+			parts := []string{}
+			if count == 1 {
+				parts = append(parts, p.part(path, 1, body, false))
+			} else {
+				parts = append(parts, p.part(path, 1, body[:5*1024*1024], false), p.part(path, 2, body[5*1024*1024:], false))
+			}
+			complete := p.request("POST", path, completion(parts...), nil, 200)
+			var receipt struct{ ETag string }
+			if err := xml.Unmarshal(complete.body, &receipt); err != nil || !strings.Contains(receipt.ETag, fmt.Sprintf("-%d", count)) {
+				t.Fatalf("completion tag: %s: %v", complete.body, err)
+			}
+			version := complete.header.Get("x-amz-version-id")
+			if version == "" {
+				t.Fatal("missing completed version")
+			}
+			destination := p.start("copied-staging", false)
+			headers := map[string]string{"x-amz-copy-source": "/uploads/staged", "x-amz-copy-source-if-match": receipt.ETag, "x-amz-copy-source-range": fmt.Sprintf("bytes=0-%d", len(body)-1)}
+			copied := p.request("PUT", destination+"&partNumber=1", nil, headers, 200)
+			var part struct{ ETag string }
+			if err := xml.Unmarshal(copied.body, &part); err != nil {
+				t.Fatal(err)
+			}
+			for _, method := range []string{"HEAD", "GET"} {
+				got := p.request(method, "/uploads/staged", nil, nil, 200)
+				if got.header.Get("ETag") != receipt.ETag {
+					t.Fatalf("%s lost completion tag: %s", method, got.header.Get("ETag"))
+				}
+				p.request(method, "/uploads/staged", nil, map[string]string{"If-None-Match": receipt.ETag}, 304)
+			}
+			for _, query := range []string{"?list-type=2", "?versions"} {
+				got := p.request("GET", "/uploads"+query, nil, nil, 200)
+				var list struct {
+					Contents []struct{ ETag string }
+					Version  []struct{ ETag string }
+				}
+				if err := xml.Unmarshal(got.body, &list); err != nil {
+					t.Fatal(err)
+				}
+				tags := []string{}
+				for _, row := range list.Contents {
+					tags = append(tags, row.ETag)
+				}
+				for _, row := range list.Version {
+					tags = append(tags, row.ETag)
+				}
+				if len(tags) != 1 || tags[0] != receipt.ETag {
+					t.Fatalf("list lost tag: %v", tags)
+				}
+			}
+			put := p.request("PUT", "/uploads/staged", []byte("replacement"), map[string]string{"If-Match": receipt.ETag}, 200)
+			if put.header.Get("ETag") == receipt.ETag {
+				t.Fatal("normal overwrite inherited multipart tag")
+			}
+			p.request("PUT", destination+"&partNumber=1", nil, headers, 412)
+			headers["x-amz-copy-source"] += "?versionId=" + version
+			p.request("PUT", destination+"&partNumber=1", nil, headers, 200)
+			head := p.request("HEAD", "/uploads/staged?versionId="+version, nil, nil, 200)
+			if head.header.Get("ETag") != receipt.ETag {
+				t.Fatal("version lost its tag")
+			}
+			p.request("POST", destination, completion(fmt.Sprintf("<Part><PartNumber>1</PartNumber><ETag>%s</ETag></Part>", part.ETag)), nil, 200)
+			if got := p.request("GET", "/uploads/copied-staging", nil, nil, 200); !bytes.Equal(got.body, body) {
+				t.Fatal("copied multipart staging bytes changed")
+			}
+		})
+	}
+}

package/src/components/s3-runtime/multipart.patch

@@ -350,3 +350,199 @@
 +		ChecksumSHA256 string      `xml:"ChecksumSHA256"`
 +	}{ETag: etag, LastModified: NewContentTime(g.timeSource.Now()), ChecksumSHA256: base64.StdEncoding.EncodeToString(digest.Sum(nil))})
 +}
+--- a/vendor/github.com/johannesboyne/gofakes3/backend.go
++++ b/vendor/github.com/johannesboyne/gofakes3/backend.go
+@@ -23,7 +23,9 @@
+ 	Size     int64
+ 	Contents io.ReadCloser
+ 	Hash     []byte
+-	Range    *ObjectRange
++	// EntityTag is the stored opaque tag; Hash remains the payload MD5.
++	EntityTag string
++	Range     *ObjectRange
+ 
+ 	// VersionID will be empty if bucket versioning has not been enabled.
+ 	VersionID VersionID
+@@ -31,6 +33,21 @@
+ 	// If versioning is enabled for the bucket, this is true if this object version
+ 	// is a delete marker.
+ 	IsDeleteMarker bool
++}
++
++// ETag returns the version-owned tag, retaining compatibility with backends
++// that only supply payload hashes.
++func (object *Object) ETag() string {
++	if object.EntityTag != "" {
++		return object.EntityTag
++	}
++	return FormatETag(object.Hash)
++}
++
++// MultipartObjectBackend atomically stores completion bytes, metadata and ETag
++// under the same conditional-write lock. The tag never enters client metadata.
++type MultipartObjectBackend interface {
++	PutMultipartObject(bucket, key string, metadata map[string]string, input io.Reader, size int64, tag string, conditions *PutConditions) (PutObjectResult, error)
+ }
+ 
+ type ObjectList struct {
+@@ -150,7 +167,8 @@
+ 
+ 	// Hash is the MD5 hash of the object content (used for ETag comparison)
+ 	// Only required if Exists is true
+-	Hash []byte
++	Hash      []byte
++	EntityTag string
+ }
+ 
+ // ObjectInfo is a deprecated alias for ConditionalObjectInfo.
+@@ -182,7 +200,10 @@
+ 		if len(expectedETag) >= 2 && expectedETag[0] == '"' && expectedETag[len(expectedETag)-1] == '"' {
+ 			expectedETag = expectedETag[1 : len(expectedETag)-1]
+ 		}
+-		actualETag := hex.EncodeToString(objectInfo.Hash)
++		actualETag := strings.Trim(objectInfo.EntityTag, "\"")
++		if actualETag == "" {
++			actualETag = hex.EncodeToString(objectInfo.Hash)
++		}
+ 		if expectedETag != actualETag {
+ 			return ErrorMessage(ErrPreconditionFailed, "The ETag does not match")
+ 		}
+--- a/vendor/github.com/johannesboyne/gofakes3/uploader.go
++++ b/vendor/github.com/johannesboyne/gofakes3/uploader.go
+@@ -496,7 +496,12 @@
+ 		input.ChecksumSHA256 = ""
+ 		input.ChecksumType = ""
+ 	}
+-	result, err := u.storage.PutObject(bucket, object, mpu.Meta, bytes.NewReader(body), int64(len(body)), input.Conditions)
++	var result PutObjectResult
++	if backend, ok := u.storage.(MultipartObjectBackend); ok {
++		result, err = backend.PutMultipartObject(bucket, object, mpu.Meta, bytes.NewReader(body), int64(len(body)), etag, input.Conditions)
++	} else {
++		result, err = u.storage.PutObject(bucket, object, mpu.Meta, bytes.NewReader(body), int64(len(body)), input.Conditions)
++	}
+ 	if err != nil {
+ 		return "", "", err
+ 	}
+--- a/vendor/github.com/johannesboyne/gofakes3/gofakes3.go
++++ b/vendor/github.com/johannesboyne/gofakes3/gofakes3.go
+@@ -535,7 +535,7 @@
+ 		w.Header().Set("x-amz-version-id", string(obj.VersionID))
+ 	}
+ 
+-	etag := `"` + hex.EncodeToString(obj.Hash) + `"`
++	etag := obj.ETag()
+ 	w.Header().Set("ETag", etag)
+ 
+ 	if r.Header.Get("If-None-Match") == etag {
+@@ -576,7 +576,16 @@
+ 		return err
+ 	}
+ 
+-	obj, err := g.storage.HeadObject(bucket, object)
++	var obj *Object
++	var err error
++	if versionID == "" {
++		obj, err = g.storage.HeadObject(bucket, object)
++	} else {
++		if g.versioned == nil {
++			return ErrNotImplemented
++		}
++		obj, err = g.versioned.HeadObjectVersion(bucket, object, versionID)
++	}
+ 	if err != nil {
+ 		return err
+ 	}
+--- a/vendor/github.com/johannesboyne/gofakes3/upload_part_copy.go
++++ b/vendor/github.com/johannesboyne/gofakes3/upload_part_copy.go
+@@ -56,7 +56,7 @@
+ 	if src.IsDeleteMarker {
+ 		return ErrInvalidArgument
+ 	}
+-	tag := FormatETag(src.Hash)
++	tag := src.ETag()
+ 	if expected := r.Header.Get("x-amz-copy-source-if-match"); expected != "" && expected != tag {
+ 		return ErrPreconditionFailed
+ 	}
+--- a/vendor/github.com/johannesboyne/gofakes3/backend/s3mem/backend.go
++++ b/vendor/github.com/johannesboyne/gofakes3/backend/s3mem/backend.go
+@@ -119,7 +119,7 @@
+ 			response.Add(&gofakes3.Content{
+ 				Key:          item.data.name,
+ 				LastModified: gofakes3.NewContentTime(item.data.lastModified),
+-				ETag:         gofakes3.FormatETag(item.data.hash),
++				ETag:         item.data.entityTag,
+ 				Size:         int64(len(item.data.body)),
+ 			})
+ 		}
+@@ -240,6 +240,16 @@
+ 	size int64,
+ 	conditions *gofakes3.PutConditions,
+ ) (result gofakes3.PutObjectResult, err error) {
++	return db.putObject(bucketName, objectName, meta, input, size, "", conditions)
++}
++
++// PutMultipartObject retains the completion tag in the same object version as
++// its bytes and uses the ordinary atomic conditional-write admission.
++func (db *Backend) PutMultipartObject(bucketName, objectName string, meta map[string]string, input io.Reader, size int64, tag string, conditions *gofakes3.PutConditions) (gofakes3.PutObjectResult, error) {
++	return db.putObject(bucketName, objectName, meta, input, size, tag, conditions)
++}
++
++func (db *Backend) putObject(bucketName, objectName string, meta map[string]string, input io.Reader, size int64, tag string, conditions *gofakes3.PutConditions) (result gofakes3.PutObjectResult, err error) {
+ 	// No need to lock the backend while we read the data into memory; it holds
+ 	// the write lock open unnecessarily, and could be blocked for an unreasonably
+ 	// long time by a connection timing out:
+@@ -272,11 +282,15 @@
+ 	}
+ 
+ 	hash := md5.Sum(bts)
++	if tag == "" {
++		tag = gofakes3.FormatETag(hash[:])
++	}
+ 
+ 	item := &bucketData{
+ 		name:         objectName,
+ 		body:         bts,
+ 		hash:         hash[:],
++		entityTag:    tag,
+ 		metadata:     meta,
+ 		lastModified: db.timeSource.Now(),
+ 	}
+@@ -557,7 +571,7 @@
+ 					IsLatest:     version == object.data,
+ 					LastModified: gofakes3.NewContentTime(version.lastModified),
+ 					Size:         int64(len(version.body)),
+-					ETag:         gofakes3.FormatETag(version.hash),
++					ETag:         version.entityTag,
+ 				}
+ 				if bucket.versioning != gofakes3.VersioningNone { // S300005
+ 					resultVer.VersionID = version.versionID
+@@ -587,8 +601,9 @@
+ 		return &gofakes3.ConditionalObjectInfo{Exists: false}, nil
+ 	}
+ 	return &gofakes3.ConditionalObjectInfo{
+-		Exists: true,
+-		Hash:   existing.data.hash,
++		Exists:    true,
++		Hash:      existing.data.hash,
++		EntityTag: existing.data.entityTag,
+ 	}, nil
+ }
+ 
+--- a/vendor/github.com/johannesboyne/gofakes3/backend/s3mem/bucket.go
++++ b/vendor/github.com/johannesboyne/gofakes3/backend/s3mem/bucket.go
+@@ -118,6 +118,7 @@
+ 	deleteMarker bool
+ 	body         []byte
+ 	hash         []byte
++	entityTag    string
+ 	metadata     map[string]string
+ }
+ 
+@@ -150,6 +151,7 @@
+ 	return &gofakes3.Object{
+ 		Name:           bi.name,
+ 		Hash:           bi.hash,
++		EntityTag:      bi.entityTag,
+ 		Metadata:       bi.metadata,
+ 		Size:           sz,
+ 		Range:          rnge,