@specific.dev/spectest 0.131.6 → 0.131.8

package/dist/components/s3.js

@@ -8,11 +8,11 @@ const S3_DOCKERFILE = `
 FROM golang:1.27.2-alpine@sha256:f92b6ef800e499660581efdabdf25d9d817a9d124eaf900924f0504e7e27e12d AS build
 RUN apk add --no-cache patch
 WORKDIR /src
-${["go.mod", "go.sum", "main.go", "metadata.patch"].map((file) => {
+${["go.mod", "go.sum", "main.go", "metadata.patch", "multipart.patch"].map((file) => {
     const data = readFileSync(new URL(`./s3-runtime/${file}`, import.meta.url)).toString("base64");
     return `RUN printf '%s' '${data}' | base64 -d > ${file}`;
 }).join("\n")}
-RUN go mod vendor && patch -p1 < metadata.patch
+RUN go mod vendor && patch -p1 < metadata.patch && patch -p1 < multipart.patch
 RUN CGO_ENABLED=0 go build -mod=vendor -trimpath -ldflags='-s -w' -o /s3-server .
 FROM scratch
 COPY --from=build /s3-server /s3-server

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@specific.dev/spectest",
-  "version": "0.131.6",
+  "version": "0.131.8",
   "description": "Spectest SDK for defining test environments in TypeScript.",
   "type": "module",
   "main": "./dist/index.js",

package/src/components/s3-runtime/README.md

@@ -16,6 +16,22 @@ uploads, multipart initiation and copies. Upstream currently drops these headers
 in `metadataHeaders`. The patch applies to the vendored, checksum-verified module;
 a failed patch stops the build. Recheck it when changing the upstream pin.
 
+`multipart.patch` adds SHA256 composite multipart integrity to the existing
+uploader: supplied part checksums are validated before storage, completion checks
+each part and its ordering, and the composite checksum survives HEAD/GET for
+reconciliation. Completion conditions use the backend's atomic conditional write.
+Overwriting an object does not inherit the previous object's checksum metadata.
+This patch supports SHA256/COMPOSITE; it does not implement every S3 checksum
+algorithm or streaming checksum trailers. It also supports UploadPartCopy from
+one backend snapshot, including closed byte ranges, exact ETag conditions,
+version selection and SHA256 part receipts. Date-based copy conditions are
+explicitly unsupported (501), rather than silently ignored.
+
+Run `./test.sh` from this directory to exercise real HTTP requests against the
+same patched sources used in the image, including corrupt retries, conditional
+completion races, abort, metadata reconciliation and unchecksummed uploads.
+The test runs in a temporary vendor directory and enables Go's race detector.
+
 From `sdk`, run the real-image contract on a Linux Docker host:
 
 ```

package/src/components/s3-runtime/main.go

@@ -11,6 +11,10 @@ import (
 )
 
 func main() {
+	log.Fatal(http.ListenAndServe(":9090", newServer()))
+}
+
+func newServer() http.Handler {
 	// VM snapshots include RAM: buckets, versions and in-flight multipart state
 	// fork together. A fresh process starts empty, as S3Mock did by default.
 	// Unlike the Bolt backend, s3mem also supports bucket versioning.
@@ -21,7 +25,7 @@ func main() {
 		gofakes3.WithAutoBucket(false),
 		gofakes3.WithLogger(gofakes3.GlobalLog()),
 	).Server()
-	log.Fatal(http.ListenAndServe(":9090", cors(server)))
+	return cors(server)
 }
 
 // Preserve S3Mock's permissive browser uploads, including arbitrary metadata

package/src/components/s3-runtime/main_test.go

@@ -0,0 +1,282 @@
+package main
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+)
+
+type s3Response struct {
+	status int
+	header http.Header
+	body   []byte
+}
+type s3Probe struct {
+	t      *testing.T
+	server *httptest.Server
+}
+
+func newProbe(t *testing.T) *s3Probe {
+	t.Helper()
+	p := &s3Probe{t, httptest.NewServer(newServer())}
+	t.Cleanup(p.server.Close)
+	p.request("PUT", "/uploads", nil, nil, 200)
+	return p
+}
+func (p *s3Probe) request(method, path string, body []byte, headers map[string]string, want int) s3Response {
+	p.t.Helper()
+	req, err := http.NewRequest(method, p.server.URL+path, bytes.NewReader(body))
+	if err != nil {
+		p.t.Fatal(err)
+	}
+	for k, v := range headers {
+		req.Header.Set(k, v)
+	}
+	res, err := p.server.Client().Do(req)
+	if err != nil {
+		p.t.Fatal(err)
+	}
+	defer res.Body.Close()
+	data, err := io.ReadAll(res.Body)
+	if err != nil {
+		p.t.Fatal(err)
+	}
+	if res.StatusCode != want {
+		p.t.Fatalf("%s %s: got %d want %d: %s", method, path, res.StatusCode, want, data)
+	}
+	return s3Response{res.StatusCode, res.Header, data}
+}
+func sum(body []byte) string {
+	digest := sha256.Sum256(body)
+	return base64.StdEncoding.EncodeToString(digest[:])
+}
+func (p *s3Probe) start(key string, checksum bool) string {
+	p.t.Helper()
+	headers := map[string]string{"Cache-Control": "private, max-age=60"}
+	if checksum {
+		headers["x-amz-checksum-algorithm"] = "SHA256"
+		headers["x-amz-checksum-type"] = "COMPOSITE"
+	}
+	res := p.request("POST", "/uploads/"+key+"?uploads", nil, headers, 200)
+	var out struct {
+		UploadID string `xml:"UploadId"`
+	}
+	if err := xml.Unmarshal(res.body, &out); err != nil || out.UploadID == "" {
+		p.t.Fatalf("initiate: %s, %v", res.body, err)
+	}
+	return "/uploads/" + key + "?uploadId=" + out.UploadID
+}
+func (p *s3Probe) part(path string, number int, body []byte, checksum bool) string {
+	p.t.Helper()
+	headers := map[string]string{}
+	if checksum {
+		headers["x-amz-checksum-sha256"] = sum(body)
+	}
+	res := p.request("PUT", fmt.Sprintf("%s&partNumber=%d", path, number), body, headers, 200)
+	if checksum && res.header.Get("x-amz-checksum-sha256") != sum(body) {
+		p.t.Fatalf("part checksum acknowledgement: %v", res.header)
+	}
+	tag := res.header.Get("ETag")
+	if tag == "" {
+		p.t.Fatal("missing ETag")
+	}
+	digest := ""
+	if checksum {
+		digest = "<ChecksumSHA256>" + sum(body) + "</ChecksumSHA256>"
+	}
+	return fmt.Sprintf("<Part><PartNumber>%d</PartNumber><ETag>%s</ETag>%s</Part>", number, tag, digest)
+}
+func completion(parts ...string) []byte {
+	return []byte("<CompleteMultipartUpload>" + strings.Join(parts, "") + "</CompleteMultipartUpload>")
+}
+
+func TestMultipartSHA256IntegrityAndReconciliation(t *testing.T) {
+	p := newProbe(t)
+	path := p.start("large", true)
+	first := bytes.Repeat([]byte("a"), 8*1024*1024)
+	last := bytes.Repeat([]byte("b"), 1024*1024)
+	part1 := p.part(path, 1, first, true)
+	// A rejected retry must not replace the already acknowledged part.
+	res := p.request("PUT", path+"&partNumber=1", []byte("corrupt"), map[string]string{"x-amz-checksum-sha256": sum(first)}, 400)
+	if !bytes.Contains(res.body, []byte("BadDigest")) {
+		t.Fatalf("corrupt part: %s", res.body)
+	}
+	part2 := p.part(path, 2, last, true)
+	for _, parts := range [][]string{{part2, part1}, {part1, part1}, {strings.Replace(part1, "<PartNumber>1", "<PartNumber>-1", 1)}, {strings.Replace(part1, sum(first), sum(last), 1), part2}, {strings.Replace(part1, "<ETag>", "<ETag>bad", 1), part2}} {
+		p.request("POST", path, completion(parts...), nil, 400)
+	}
+	p.request("POST", path, completion(part1, part2), map[string]string{"x-amz-checksum-type": "FULL_OBJECT"}, 400)
+	p.request("POST", path, completion(part1, part2), map[string]string{"x-amz-checksum-sha256": "wrong"}, 400)
+	a, b := sha256.Sum256(first), sha256.Sum256(last)
+	expected := sum(append(a[:], b[:]...)) + "-2"
+	res = p.request("POST", path, completion(part1, part2), map[string]string{"If-None-Match": "*", "x-amz-checksum-type": "COMPOSITE", "x-amz-checksum-sha256": expected}, 200)
+	var completed struct {
+		ChecksumSHA256 string
+		ChecksumType   string
+	}
+	if err := xml.Unmarshal(res.body, &completed); err != nil {
+		t.Fatal(err)
+	}
+	if completed.ChecksumSHA256 != expected || completed.ChecksumType != "COMPOSITE" {
+		t.Fatalf("complete: %s", res.body)
+	}
+	// Reconcile using retained metadata after an unobserved completion response.
+	head := p.request("HEAD", "/uploads/large", nil, map[string]string{"x-amz-checksum-mode": "ENABLED"}, 200)
+	if head.header.Get("x-amz-checksum-sha256") != expected || head.header.Get("x-amz-checksum-type") != "COMPOSITE" || head.header.Get("Content-Length") != fmt.Sprint(len(first)+len(last)) || head.header.Get("Cache-Control") != "private, max-age=60" {
+		t.Fatalf("reconciliation: %v", head.header)
+	}
+	got := p.request("GET", "/uploads/large", nil, nil, 200)
+	if !bytes.Equal(got.body, append(first, last...)) {
+		t.Fatal("completed bytes differ")
+	}
+	p.request("PUT", path+"&partNumber=1", first, nil, 404)
+	// An ordinary overwrite must not inherit an obsolete composite checksum.
+	p.request("PUT", "/uploads/large", []byte("replacement"), nil, 200)
+	head = p.request("HEAD", "/uploads/large", nil, nil, 200)
+	if head.header.Get("x-amz-checksum-sha256") != "" || head.header.Get("x-amz-checksum-type") != "" {
+		t.Fatal("overwrite retained stale checksum")
+	}
+}
+
+func TestMultipartConditionalCompletionIsAtomic(t *testing.T) {
+	p := newProbe(t)
+	paths := []string{p.start("same", true), p.start("same", true)}
+	parts := []string{p.part(paths[0], 1, []byte("first"), true), p.part(paths[1], 1, []byte("second"), true)}
+	statuses := make(chan int, 2)
+	var wg sync.WaitGroup
+	for i := range paths {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			req, _ := http.NewRequest("POST", p.server.URL+paths[i], bytes.NewReader(completion(parts[i])))
+			req.Header.Set("If-None-Match", "*")
+			res, err := p.server.Client().Do(req)
+			if err != nil {
+				statuses <- 0
+				return
+			}
+			defer res.Body.Close()
+			io.Copy(io.Discard, res.Body)
+			statuses <- res.StatusCode
+		}(i)
+	}
+	wg.Wait()
+	close(statuses)
+	counts := map[int]int{}
+	for s := range statuses {
+		counts[s]++
+	}
+	if counts[200] != 1 || counts[412] != 1 {
+		t.Fatalf("competing creates: %v", counts)
+	}
+	body := p.request("GET", "/uploads/same", nil, nil, 200).body
+	if string(body) != "first" && string(body) != "second" {
+		t.Fatalf("bad winner: %s", body)
+	}
+}
+
+func TestMultipartAbortAndLegacyUpload(t *testing.T) {
+	p := newProbe(t)
+	path := p.start("aborted", true)
+	part := p.part(path, 1, []byte("abc"), true)
+	p.request("DELETE", path, nil, nil, 204)
+	p.request("POST", path, completion(part), nil, 404)
+	p.request("HEAD", "/uploads/aborted", nil, nil, 404)
+	path = p.start("legacy", false)
+	part = p.part(path, 1, []byte("legacy"), false)
+	p.request("POST", path, completion(part), nil, 200)
+	if got := p.request("GET", "/uploads/legacy", nil, nil, 200); string(got.body) != "legacy" {
+		t.Fatal("legacy multipart changed")
+	}
+}
+
+func TestUploadPartCopyRetainsExactSourceAndRejectsStaleRetry(t *testing.T) {
+	p := newProbe(t)
+	source := bytes.Repeat([]byte("0123456789"), 600000)
+	put := p.request("PUT", "/uploads/staging%20%2B%25.csv", source, nil, 200)
+	path := p.start("copied", false)
+	headers := map[string]string{"x-amz-copy-source": "/uploads/staging%20%2B%25.csv", "x-amz-copy-source-if-match": put.header.Get("ETag"), "x-amz-copy-source-range": "bytes=3-5242882"}
+	res := p.request("PUT", path+"&partNumber=1", nil, headers, 200)
+	var part struct {
+		XMLName      xml.Name
+		ETag         string
+		LastModified string
+	}
+	if err := xml.Unmarshal(res.body, &part); err != nil || part.XMLName.Local != "CopyPartResult" || part.ETag == "" || part.LastModified == "" {
+		t.Fatalf("copy part receipt: %s: %v", res.body, err)
+	}
+	first := fmt.Sprintf("<Part><PartNumber>1</PartNumber><ETag>%s</ETag></Part>", part.ETag)
+	headers["x-amz-copy-source-if-match"] = `"stale"`
+	p.request("PUT", path+"&partNumber=1", nil, headers, 412)
+	headers["x-amz-copy-source-if-match"] = put.header.Get("ETag")
+	for _, invalid := range []string{"bytes=6000000-6000001", "bytes=3-6000000", "bytes=5-3", "bytes=-3", "bytes=3-", "bytes=0-1,3-4"} {
+		headers["x-amz-copy-source-range"] = invalid
+		p.request("PUT", path+"&partNumber=1", nil, headers, 416)
+	}
+	headers["x-amz-copy-source-range"] = "bytes=5242883-5999999"
+	res = p.request("PUT", path+"&partNumber=2", nil, headers, 200)
+	if err := xml.Unmarshal(res.body, &part); err != nil {
+		t.Fatal(err)
+	}
+	second := fmt.Sprintf("<Part><PartNumber>2</PartNumber><ETag>%s</ETag></Part>", part.ETag)
+	p.request("POST", path, completion(first, second), nil, 200)
+	got := p.request("GET", "/uploads/copied", nil, nil, 200)
+	if !bytes.Equal(got.body, source[3:]) {
+		t.Fatal("copy changed the retained byte range")
+	}
+	p.request("PUT", path+"&partNumber=2", nil, headers, 404)
+}
+
+func TestUploadPartCopyVersionChecksumAndAbort(t *testing.T) {
+	p := newProbe(t)
+	p.request("PUT", "/uploads?versioning", []byte(`<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>`), nil, 200)
+	original := []byte("retained original version")
+	source := p.request("PUT", "/uploads/versioned", original, nil, 200)
+	version := source.header.Get("x-amz-version-id")
+	if version == "" {
+		t.Fatal("source has no version")
+	}
+	p.request("PUT", "/uploads/versioned", []byte("replacement"), nil, 200)
+	path := p.start("version-copy", true)
+	headers := map[string]string{"x-amz-copy-source": "/uploads/versioned?versionId=" + version, "x-amz-copy-source-if-match": source.header.Get("ETag")}
+	res := p.request("PUT", path+"&partNumber=1", nil, headers, 200)
+	var part struct {
+		ETag           string
+		ChecksumSHA256 string
+	}
+	if err := xml.Unmarshal(res.body, &part); err != nil {
+		t.Fatal(err)
+	}
+	if part.ChecksumSHA256 != sum(original) {
+		t.Fatal("copied version checksum differs")
+	}
+	headers["x-amz-copy-source-if-none-match"] = source.header.Get("ETag")
+	p.request("PUT", path+"&partNumber=1", nil, headers, 412)
+	delete(headers, "x-amz-copy-source-if-none-match")
+	headers["x-amz-copy-source-if-modified-since"] = "Sat, 31 Oct 2026 00:00:00 GMT"
+	p.request("PUT", path+"&partNumber=1", nil, headers, 501)
+	delete(headers, "x-amz-copy-source-if-modified-since")
+	descriptor := fmt.Sprintf("<Part><PartNumber>1</PartNumber><ETag>%s</ETag><ChecksumSHA256>%s</ChecksumSHA256></Part>", part.ETag, part.ChecksumSHA256)
+	digest := sha256.Sum256(original)
+	composite := sum(digest[:]) + "-1"
+	p.request("POST", path, completion(descriptor), map[string]string{"x-amz-checksum-type": "COMPOSITE", "x-amz-checksum-sha256": composite}, 200)
+	got := p.request("GET", "/uploads/version-copy", nil, nil, 200)
+	if !bytes.Equal(got.body, original) || got.header.Get("x-amz-checksum-sha256") != composite {
+		t.Fatal("versioned copy bytes or composite receipt changed")
+	}
+	path = p.start("aborted-copy", false)
+	p.request("DELETE", path, nil, nil, 204)
+	p.request("PUT", path+"&partNumber=1", nil, headers, 404)
+	headers["x-amz-copy-source"] = "/uploads/missing"
+	path = p.start("missing-copy", false)
+	p.request("PUT", path+"&partNumber=1", nil, headers, 404)
+	p.request("DELETE", path, nil, nil, 204)
+}