@specific.dev/spectest 0.131.6 → 0.131.8

package/src/components/s3-runtime/multipart.patch

@@ -0,0 +1,352 @@
+--- a/vendor/github.com/johannesboyne/gofakes3/gofakes3.go
++++ b/vendor/github.com/johannesboyne/gofakes3/gofakes3.go
+@@ -933,6 +933,14 @@
+ 		return err
+ 	}
+ 
++	if meta["X-Amz-Checksum-Algorithm"] == "SHA256" {
++		if kind := meta["X-Amz-Checksum-Type"]; kind != "" && kind != "COMPOSITE" {
++			return ErrInvalidArgument
++		}
++		meta["X-Amz-Checksum-Type"] = "COMPOSITE"
++		w.Header().Set("x-amz-checksum-algorithm", "SHA256")
++		w.Header().Set("x-amz-checksum-type", "COMPOSITE")
++	}
+ 	uploadID, err := g.uploader.CreateMultipartUpload(bucket, object, meta)
+ 	if err != nil {
+ 		return err
+@@ -983,12 +991,20 @@
+ 		}
+ 	}
+ 
++	var checksum *sha256PartReader
++	if expected := r.Header.Get("x-amz-checksum-sha256"); expected != "" {
++		checksum = newSHA256PartReader(rdr, expected)
++		rdr = checksum
++	}
+ 	etag, err := g.uploader.UploadPart(bucket, object, uploadID, int(partNumber), r.ContentLength, rdr)
+ 	if err != nil {
+ 		return err
+ 	}
+ 
+ 	w.Header().Add("ETag", etag)
++	if checksum != nil {
++		w.Header().Set("x-amz-checksum-sha256", checksum.sum())
++	}
+ 	return nil
+ }
+ 
+@@ -1009,6 +1025,13 @@
+ 		return err
+ 	}
+ 
++	var err error
++	in.Conditions, err = parsePutConditions(r.Header)
++	if err != nil {
++		return err
++	}
++	in.ChecksumSHA256 = r.Header.Get("x-amz-checksum-sha256")
++	in.ChecksumType = r.Header.Get("x-amz-checksum-type")
+ 	versionID, etag, err := g.uploader.CompleteMultipartUpload(bucket, object, uploadID, &in)
+ 	if err != nil {
+ 		return err
+@@ -1031,10 +1054,12 @@
+ 	}
+ 
+ 	return g.xmlEncoder(w).Encode(&CompleteMultipartUploadResult{
+-		ETag:     etag,
+-		Bucket:   bucket,
+-		Key:      object,
+-		Location: location,
++		ETag:           etag,
++		Bucket:         bucket,
++		Key:            object,
++		Location:       location,
++		ChecksumSHA256: in.ChecksumSHA256,
++		ChecksumType:   in.ChecksumType,
+ 	})
+ }
+ 
+--- a/vendor/github.com/johannesboyne/gofakes3/uploader.go
++++ b/vendor/github.com/johannesboyne/gofakes3/uploader.go
+@@ -3,6 +3,8 @@
+ import (
+ 	"bytes"
+ 	"crypto/md5"
++	"crypto/sha256"
++	"encoding/base64"
+ 	"encoding/hex"
+ 	"fmt"
+ 	"io"
+@@ -428,13 +430,26 @@
+ 		return "", "", ErrInvalidPart
+ 	}
+ 
+-	if !input.partsAreSorted() {
++	if len(input.Parts) == 0 || !input.partsAreSorted() {
+ 		return "", "", ErrInvalidPartOrder
+ 	}
+ 
+ 	var size int64
+ 
+-	for _, inPart := range input.Parts {
++	composite := sha256.New()
++	withSHA256 := mpu.Meta["X-Amz-Checksum-Algorithm"] == "SHA256"
++	if withSHA256 && input.ChecksumType != "" && input.ChecksumType != "COMPOSITE" {
++		return "", "", ErrBadDigest
++	}
++	previous := 0
++	for i, inPart := range input.Parts {
++		if inPart.PartNumber <= previous {
++			return "", "", ErrInvalidPartOrder
++		}
++		previous = inPart.PartNumber
++		if withSHA256 && inPart.PartNumber != i+1 {
++			return "", "", ErrInvalidPartOrder
++		}
+ 		if inPart.PartNumber >= mpuPartsLen || mpu.parts[inPart.PartNumber] == nil {
+ 			return "", "", ErrorMessagef(ErrInvalidPart, "unexpected part number %d in complete request", inPart.PartNumber)
+ 		}
+@@ -444,6 +459,13 @@
+ 			return "", "", ErrorMessagef(ErrInvalidPart, "unexpected part etag for number %d in complete request", inPart.PartNumber)
+ 		}
+ 
++		if withSHA256 {
++			digest := sha256.Sum256(upPart.Body)
++			if inPart.ChecksumSHA256 != base64.StdEncoding.EncodeToString(digest[:]) {
++				return "", "", ErrBadDigest
++			}
++			composite.Write(digest[:])
++		}
+ 		size += int64(len(upPart.Body))
+ 	}
+ 
+@@ -461,7 +483,20 @@
+ 
+ 	etag = fmt.Sprintf(`"%s-%d"`, hex.EncodeToString(hash.Sum(nil)), len(input.Parts))
+ 
+-	result, err := u.storage.PutObject(bucket, object, mpu.Meta, bytes.NewReader(body), int64(len(body)), nil)
++	if withSHA256 {
++		checksum := fmt.Sprintf("%s-%d", base64.StdEncoding.EncodeToString(composite.Sum(nil)), len(input.Parts))
++		if input.ChecksumSHA256 != "" && input.ChecksumSHA256 != checksum {
++			return "", "", ErrBadDigest
++		}
++		input.ChecksumSHA256 = checksum
++		input.ChecksumType = "COMPOSITE"
++		mpu.Meta["X-Amz-Checksum-Sha256"] = checksum
++		mpu.Meta["X-Amz-Checksum-Type"] = "COMPOSITE"
++	} else {
++		input.ChecksumSHA256 = ""
++		input.ChecksumType = ""
++	}
++	result, err := u.storage.PutObject(bucket, object, mpu.Meta, bytes.NewReader(body), int64(len(body)), input.Conditions)
+ 	if err != nil {
+ 		return "", "", err
+ 	}
+--- a/vendor/github.com/johannesboyne/gofakes3/messages.go
++++ b/vendor/github.com/johannesboyne/gofakes3/messages.go
+@@ -48,12 +48,16 @@
+ }
+ 
+ type CompletedPart struct {
+-	PartNumber int    `xml:"PartNumber"`
+-	ETag       string `xml:"ETag"`
++	ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
++	PartNumber     int    `xml:"PartNumber"`
++	ETag           string `xml:"ETag"`
+ }
+ 
+ type CompleteMultipartUploadRequest struct {
+-	Parts []CompletedPart `xml:"Part"`
++	Conditions     *PutConditions  `xml:"-"`
++	ChecksumSHA256 string          `xml:"-"`
++	ChecksumType   string          `xml:"-"`
++	Parts          []CompletedPart `xml:"Part"`
+ }
+ 
+ func (c CompleteMultipartUploadRequest) partsAreSorted() bool {
+@@ -70,10 +74,12 @@
+ }
+ 
+ type CompleteMultipartUploadResult struct {
+-	Location string `xml:"Location"`
+-	Bucket   string `xml:"Bucket"`
+-	Key      string `xml:"Key"`
+-	ETag     string `xml:"ETag"`
++	ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
++	ChecksumType   string `xml:"ChecksumType,omitempty"`
++	Location       string `xml:"Location"`
++	Bucket         string `xml:"Bucket"`
++	Key            string `xml:"Key"`
++	ETag           string `xml:"ETag"`
+ }
+ 
+ type Content struct {
+--- a/vendor/github.com/johannesboyne/gofakes3/checksum.go
++++ b/vendor/github.com/johannesboyne/gofakes3/checksum.go
+@@ -0,0 +1,32 @@
++package gofakes3
++
++import (
++	"crypto/sha256"
++	"encoding/base64"
++	"hash"
++	"io"
++)
++
++// Validate before UploadPart commits bytes to the existing multipart store.
++type sha256PartReader struct {
++	input    io.Reader
++	digest   hash.Hash
++	expected string
++}
++
++func newSHA256PartReader(input io.Reader, expected string) *sha256PartReader {
++	return &sha256PartReader{input: input, digest: sha256.New(), expected: expected}
++}
++
++func (r *sha256PartReader) sum() string {
++	return base64.StdEncoding.EncodeToString(r.digest.Sum(nil))
++}
++
++func (r *sha256PartReader) Read(p []byte) (int, error) {
++	n, err := r.input.Read(p)
++	r.digest.Write(p[:n])
++	if err == io.EOF && r.sum() != r.expected {
++		return n, ErrBadDigest
++	}
++	return n, err
++}
+--- a/vendor/github.com/johannesboyne/gofakes3/backend.go
++++ b/vendor/github.com/johannesboyne/gofakes3/backend.go
+@@ -3,6 +3,7 @@
+ import (
+ 	"encoding/hex"
+ 	"errors"
++	"strings"
+ 
+ 	"io"
+ 	"time"
+@@ -434,6 +435,11 @@
+ 	// carry over metadata if it exists
+ 	if existingObj != nil {
+ 		for k, v := range existingObj.Metadata {
++			// Integrity metadata belongs to the bytes being written, never a
++			// previous version of this key. CopyObject supplies its own metadata.
++			if strings.HasPrefix(k, "X-Amz-Checksum-") {
++				continue
++			}
+ 			// new metadata overwrites old but keep the rest
+ 			// TODO: check how metadata can be deleted?!
+ 			if _, ok := meta[k]; !ok {
+--- a/vendor/github.com/johannesboyne/gofakes3/gofakes3.go
++++ b/vendor/github.com/johannesboyne/gofakes3/gofakes3.go
+@@ -960,6 +960,10 @@
+ 		return ErrInvalidPart
+ 	}
+ 
++	if r.Header.Get("x-amz-copy-source") != "" {
++		return g.copyMultipartUploadPart(bucket, object, uploadID, int(partNumber), w, r)
++	}
++
+ 	size, err := strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64)
+ 	if err != nil || size <= 0 {
+ 		return ErrMissingContentLength
+--- /dev/null
++++ b/vendor/github.com/johannesboyne/gofakes3/upload_part_copy.go
+@@ -0,0 +1,95 @@
++package gofakes3
++
++import (
++	"crypto/sha256"
++	"encoding/base64"
++	"encoding/xml"
++	"io"
++	"net/http"
++	"net/url"
++	"strings"
++)
++
++// Copy from one backend snapshot: a separate HEAD followed by GET could race
++// an overwrite and copy bytes that no longer match the caller's condition.
++func (g *GoFakeS3) copyMultipartUploadPart(bucket, object string, uploadID UploadID, number int, w http.ResponseWriter, r *http.Request) error {
++	source, query, _ := strings.Cut(r.Header.Get("x-amz-copy-source"), "?")
++	source, err := url.PathUnescape(source)
++	if err != nil {
++		return ErrInvalidArgument
++	}
++	parts := strings.SplitN(strings.TrimPrefix(source, "/"), "/", 2)
++	if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
++		return ErrInvalidArgument
++	}
++	params, err := url.ParseQuery(query)
++	if err != nil {
++		return ErrInvalidArgument
++	}
++	for key := range params {
++		if key != "versionId" {
++			return ErrInvalidArgument
++		}
++	}
++	// Do not silently ignore conditions this compatibility subset cannot apply.
++	for _, key := range []string{"x-amz-copy-source-if-modified-since", "x-amz-copy-source-if-unmodified-since"} {
++		if r.Header.Get(key) != "" {
++			return ErrNotImplemented
++		}
++	}
++	var src *Object
++	if version := params.Get("versionId"); version != "" {
++		if g.versioned == nil {
++			return ErrNotImplemented
++		}
++		src, err = g.versioned.GetObjectVersion(parts[0], parts[1], VersionID(version), nil)
++	} else {
++		src, err = g.storage.GetObject(parts[0], parts[1], nil)
++	}
++	if err != nil {
++		return err
++	}
++	if src == nil || src.Contents == nil {
++		return ErrInternal
++	}
++	defer src.Contents.Close()
++	if src.IsDeleteMarker {
++		return ErrInvalidArgument
++	}
++	tag := FormatETag(src.Hash)
++	if expected := r.Header.Get("x-amz-copy-source-if-match"); expected != "" && expected != tag {
++		return ErrPreconditionFailed
++	}
++	if expected := r.Header.Get("x-amz-copy-source-if-none-match"); expected != "" && expected == tag {
++		return ErrPreconditionFailed
++	}
++	start, size := int64(0), src.Size
++	if value := r.Header.Get("x-amz-copy-source-range"); value != "" {
++		span, err := parseRangeHeader(value)
++		if err != nil || span == nil || span.FromEnd || span.End == RangeNoEnd || span.End >= src.Size {
++			return ErrInvalidRange
++		}
++		start, size = span.Start, span.End-span.Start+1
++	}
++	if size <= 0 {
++		return ErrInvalidRange
++	}
++	if _, err := io.CopyN(io.Discard, src.Contents, start); err != nil {
++		return err
++	}
++	digest := sha256.New()
++	body := io.TeeReader(io.LimitReader(src.Contents, size), digest)
++	etag, err := g.uploader.UploadPart(bucket, object, uploadID, number, size, body)
++	if err != nil {
++		return err
++	}
++	if src.VersionID != "" {
++		w.Header().Set("x-amz-copy-source-version-id", string(src.VersionID))
++	}
++	return g.xmlEncoder(w).Encode(struct {
++		XMLName        xml.Name    `xml:"CopyPartResult"`
++		ETag           string      `xml:"ETag"`
++		LastModified   ContentTime `xml:"LastModified"`
++		ChecksumSHA256 string      `xml:"ChecksumSHA256"`
++	}{ETag: etag, LastModified: NewContentTime(g.timeSource.Now()), ChecksumSHA256: base64.StdEncoding.EncodeToString(digest.Sum(nil))})
++}

package/src/components/s3-runtime/test.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+# Exercise the exact vendored compatibility patches used by the runtime image.
+set -eu
+source_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+test_dir=$(mktemp -d)
+trap 'rm -rf "$test_dir"' EXIT HUP INT TERM
+cp "$source_dir"/*.go "$source_dir/go.mod" "$source_dir/go.sum" "$source_dir"/*.patch "$test_dir/"
+cd "$test_dir"
+go mod vendor
+patch -p1 < metadata.patch
+patch -p1 < multipart.patch
+go test -mod=vendor -race ./...

package/src/components/s3.ts

@@ -52,11 +52,11 @@ const S3_DOCKERFILE = `
 FROM golang:1.27.2-alpine@sha256:f92b6ef800e499660581efdabdf25d9d817a9d124eaf900924f0504e7e27e12d AS build
 RUN apk add --no-cache patch
 WORKDIR /src
-${["go.mod", "go.sum", "main.go", "metadata.patch"].map((file) => {
+${["go.mod", "go.sum", "main.go", "metadata.patch", "multipart.patch"].map((file) => {
   const data = readFileSync(new URL(`./s3-runtime/${file}`, import.meta.url)).toString("base64");
   return `RUN printf '%s' '${data}' | base64 -d > ${file}`;
 }).join("\n")}
-RUN go mod vendor && patch -p1 < metadata.patch
+RUN go mod vendor && patch -p1 < metadata.patch && patch -p1 < multipart.patch
 RUN CGO_ENABLED=0 go build -mod=vendor -trimpath -ldflags='-s -w' -o /s3-server .
 FROM scratch
 COPY --from=build /s3-server /s3-server