cdk-ecr-deployment 3.3.0 → 4.0.0

package/lambda-src/internal/tarfile/reader.go

@@ -1,179 +0,0 @@
-// Taken from https://github.com/containers/image
-// Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-
-package tarfile
-
-import (
-	"archive/tar"
-	"encoding/json"
-	"io"
-	"os"
-	"path"
-
-	"cdk-ecr-deployment-handler/internal/iolimits"
-
-	"github.com/containers/image/v5/docker/reference"
-	"github.com/pkg/errors"
-)
-
-// S3FileReader is a ((docker save)-formatted) tar archive that allows random access to any component.
-type S3FileReader struct {
-	// None of the fields below are modified after the archive is created, until .Close();
-	// this allows concurrent readers of the same archive.
-	s3file   *S3File
-	Manifest []ManifestItem // Guaranteed to exist after the archive is created.
-}
-
-// newReader creates a Reader for the specified path and removeOnClose flag.
-// The caller should call .Close() on the returned archive when done.
-func NewS3FileReader(s3file *S3File) (*S3FileReader, error) {
-	if s3file == nil {
-		return nil, errors.New("s3.tarfile.S3FileReader can't be nil")
-	}
-
-	// This is a valid enough archive, except Manifest is not yet filled.
-	r := &S3FileReader{s3file: s3file}
-
-	// We initialize Manifest immediately when constructing the Reader instead
-	// of later on-demand because every caller will need the data, and because doing it now
-	// removes the need to synchronize the access/creation of the data if the archive is later
-	// used from multiple goroutines to access different images.
-
-	// FIXME? Do we need to deal with the legacy format?
-	bytes, err := r.readTarComponent(manifestFileName, iolimits.MegaByte)
-	if err != nil {
-		return nil, err
-	}
-	if err := json.Unmarshal(bytes, &r.Manifest); err != nil {
-		return nil, errors.Wrap(err, "Error decoding tar manifest.json")
-	}
-
-	return r, nil
-}
-
-// Close removes resources associated with an initialized Reader, if any.
-func (r *S3FileReader) Close() error {
-	return r.s3file.Close()
-}
-
-// ChooseManifestItem selects a manifest item from r.Manifest matching (ref, sourceIndex), one or
-// both of which should be (nil, -1).
-// On success, it returns the manifest item and an index of the matching tag, if a tag was used
-// for matching; the index is -1 if a tag was not used.
-func (r *S3FileReader) ChooseManifestItem(ref reference.NamedTagged, sourceIndex int) (*ManifestItem, int, error) {
-	switch {
-	case ref != nil && sourceIndex != -1:
-		return nil, -1, errors.Errorf("Internal error: Cannot have both ref %s and source index @%d",
-			ref.String(), sourceIndex)
-
-	case ref != nil:
-		refString := ref.String()
-		for i := range r.Manifest {
-			for tagIndex, tag := range r.Manifest[i].RepoTags {
-				parsedTag, err := reference.ParseNormalizedNamed(tag)
-				if err != nil {
-					return nil, -1, errors.Wrapf(err, "Invalid tag %#v in manifest.json item @%d", tag, i)
-				}
-				if parsedTag.String() == refString {
-					return &r.Manifest[i], tagIndex, nil
-				}
-			}
-		}
-		return nil, -1, errors.Errorf("Tag %#v not found", refString)
-
-	case sourceIndex != -1:
-		if sourceIndex >= len(r.Manifest) {
-			return nil, -1, errors.Errorf("Invalid source index @%d, only %d manifest items available",
-				sourceIndex, len(r.Manifest))
-		}
-		return &r.Manifest[sourceIndex], -1, nil
-
-	default:
-		if len(r.Manifest) != 1 {
-			return nil, -1, errors.Errorf("Unexpected tar manifest.json: expected 1 item, got %d", len(r.Manifest))
-		}
-		return &r.Manifest[0], -1, nil
-	}
-}
-
-// tarReadCloser is a way to close the backing file of a tar.Reader when the user no longer needs the tar component.
-type tarReadCloser struct {
-	*tar.Reader
-}
-
-func (t *tarReadCloser) Close() error {
-	return nil
-}
-
-// openTarComponent returns a ReadCloser for the specific file within the archive.
-// This is linear scan; we assume that the tar file will have a fairly small amount of files (~layers),
-// and that filesystem caching will make the repeated seeking over the (uncompressed) tarPath cheap enough.
-// It is safe to call this method from multiple goroutines simultaneously.
-// The caller should call .Close() on the returned stream.
-func (r *S3FileReader) openTarComponent(componentPath string) (io.ReadCloser, error) {
-	// We must clone at here because we need to make sure each tar reader must read from the beginning.
-	// And the internal rcache should be shared.
-	f := r.s3file.Clone()
-	tarReader, header, err := findTarComponent(f, componentPath)
-	if err != nil {
-		return nil, err
-	}
-	if header == nil {
-		return nil, os.ErrNotExist
-	}
-	if header.FileInfo().Mode()&os.ModeType == os.ModeSymlink { // FIXME: untested
-		// We follow only one symlink; so no loops are possible.
-		if _, err := f.Seek(0, io.SeekStart); err != nil {
-			return nil, err
-		}
-		// The new path could easily point "outside" the archive, but we only compare it to existing tar headers without extracting the archive,
-		// so we don't care.
-		tarReader, header, err = findTarComponent(f, path.Join(path.Dir(componentPath), header.Linkname))
-		if err != nil {
-			return nil, err
-		}
-		if header == nil {
-			return nil, os.ErrNotExist
-		}
-	}
-
-	if !header.FileInfo().Mode().IsRegular() {
-		return nil, errors.Errorf("Error reading tar archive component %s: not a regular file", header.Name)
-	}
-	return &tarReadCloser{Reader: tarReader}, nil
-}
-
-// findTarComponent returns a header and a reader matching componentPath within inputFile,
-// or (nil, nil, nil) if not found.
-func findTarComponent(inputFile io.Reader, componentPath string) (*tar.Reader, *tar.Header, error) {
-	t := tar.NewReader(inputFile)
-	componentPath = path.Clean(componentPath)
-	for {
-		h, err := t.Next()
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return nil, nil, err
-		}
-		if path.Clean(h.Name) == componentPath {
-			return t, h, nil
-		}
-	}
-	return nil, nil, nil
-}
-
-// readTarComponent returns full contents of componentPath.
-// It is safe to call this method from multiple goroutines simultaneously.
-func (r *S3FileReader) readTarComponent(path string, limit int) ([]byte, error) {
-	file, err := r.openTarComponent(path)
-	if err != nil {
-		return nil, errors.Wrapf(err, "Error loading tar component %s", path)
-	}
-	defer file.Close()
-	bytes, err := iolimits.ReadAtMost(file, limit)
-	if err != nil {
-		return nil, err
-	}
-	return bytes, nil
-}

package/lambda-src/internal/tarfile/reader_test.go

@@ -1,31 +0,0 @@
-// Taken from https://github.com/containers/image
-// Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-
-package tarfile
-
-import (
-	"context"
-	"log"
-	"testing"
-
-	"github.com/aws/aws-sdk-go-v2/config"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestNewS3FileReader(t *testing.T) {
-	t.Skip()
-	cfg, err := config.LoadDefaultConfig(context.TODO())
-	assert.NoError(t, err)
-
-	s3uri, _ := ParseS3Uri("s3://cdk-ecr-deployment/nginx.tar")
-
-	f, err := NewS3File(cfg, *s3uri)
-	assert.NoError(t, err)
-
-	log.Printf("file size: %d", f.Size())
-
-	reader, err := NewS3FileReader(f)
-	assert.NoError(t, err)
-
-	log.Printf("%+v", reader.Manifest)
-}

package/lambda-src/internal/tarfile/s3file.go

@@ -1,363 +0,0 @@
-// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-package tarfile
-
-import (
-	"cdk-ecr-deployment-handler/internal/iolimits"
-	"context"
-	"fmt"
-	"io"
-	"strings"
-	"sync"
-
-	"github.com/pkg/errors"
-
-	"github.com/sirupsen/logrus"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/golang/groupcache/lru"
-)
-
-const S3Prefix = "s3://"
-
-type S3Uri struct {
-	Bucket string
-	Key    string
-}
-
-func ParseS3Uri(s string) (*S3Uri, error) {
-	if !strings.HasPrefix(s, S3Prefix) {
-		return nil, fmt.Errorf("s3 uri must begin with %v", S3Prefix)
-	}
-	s = strings.TrimPrefix(s, S3Prefix)
-	parts := strings.SplitN(s, "/", 2)
-	if len(parts) == 1 {
-		return &S3Uri{
-			Bucket: parts[0],
-			Key:    "",
-		}, nil
-	}
-	return &S3Uri{
-		Bucket: parts[0],
-		Key:    parts[1],
-	}, nil
-}
-
-type S3File struct {
-	s3uri  S3Uri
-	client *s3.Client
-	i      int64       // current reading index
-	size   int64       // the size of the s3 object
-	rcache *BlockCache // read cache
-}
-
-// Len returns the number of bytes of the unread portion of the s3 object
-func (f *S3File) Len() int64 {
-	if f.i >= f.size {
-		return 0
-	}
-	return f.size - f.i
-}
-
-// Size returns the original length of the s3 object
-func (f *S3File) Size() int64 {
-	return f.size
-}
-
-// func (f *S3File) Read(b []byte) (n int, err error) {
-// 	logrus.Debugf("S3File: Read %d bytes", len(b))
-
-// 	if f.i >= f.size {
-// 		return 0, io.EOF
-// 	}
-// 	out, err := f.client.GetObject(context.TODO(), &s3.GetObjectInput{
-// 		Bucket: &f.s3uri.Bucket,
-// 		Key:    &f.s3uri.Key,
-// 		Range:  aws.String(fmt.Sprintf("bytes=%d-%d", f.i, f.i+int64(len(b))-1)),
-// 	})
-// 	if err != nil {
-// 		return 0, err
-// 	}
-// 	defer out.Body.Close()
-
-// 	n, err = out.Body.Read(b)
-// 	f.i += int64(n)
-// 	if err == io.EOF {
-// 		return n, nil // e is EOF, so return nil explicitly
-// 	}
-// 	return
-// }
-
-func (f *S3File) onCacheMiss(block *Block) (err error) {
-	if f.client == nil {
-		return errors.New("S3File: api client is nil, did you close the file?")
-	}
-	bid := block.Id
-	out, err := f.client.GetObject(context.TODO(), &s3.GetObjectInput{
-		Bucket: &f.s3uri.Bucket,
-		Key:    &f.s3uri.Key,
-		Range:  aws.String(fmt.Sprintf("bytes=%d-%d", bid*iolimits.BlockSize, (bid+1)*iolimits.BlockSize-1)),
-	})
-	if err != nil {
-		return err
-	}
-	defer out.Body.Close()
-
-	i, n := 0, 0
-	for i < iolimits.BlockSize {
-		n, err = out.Body.Read(block.Buf[i:iolimits.BlockSize])
-		i += n
-		if err != nil {
-			break
-		}
-	}
-	if err == io.EOF {
-		return nil
-	}
-	return err
-}
-
-// Read implements the io.Reader interface.
-func (f *S3File) Read(b []byte) (n int, err error) {
-	logrus.Debugf("S3File: Read %d bytes", len(b))
-
-	if f.i >= f.size {
-		return 0, io.EOF
-	}
-	if f.rcache == nil {
-		return 0, errors.New("S3File: rcache is nil, did you close the file?")
-	}
-	buf, err := f.rcache.Read(f.i, f.i+int64(len(b)), f.onCacheMiss)
-	if err != nil {
-		return 0, err
-	}
-	n = copy(b, buf)
-	f.i += int64(n)
-	return n, nil
-}
-
-// ReadAt implements the io.ReaderAt interface.
-func (f *S3File) ReadAt(b []byte, off int64) (n int, err error) {
-	logrus.Debugf("S3File: ReadAt %d bytes %d offset", len(b), off)
-
-	if off < 0 {
-		return 0, errors.New("S3File: negative offset")
-	}
-	if off >= f.size {
-		return 0, io.EOF
-	}
-	if f.rcache == nil {
-		return 0, errors.New("S3File: rcache is nil, did you close the file?")
-	}
-	buf, err := f.rcache.Read(off, off+int64(len(b)), f.onCacheMiss)
-	if err != nil {
-		return 0, err
-	}
-	return copy(b, buf), nil
-}
-
-// Seek implements the io.Seeker interface.
-func (f *S3File) Seek(offset int64, whence int) (int64, error) {
-	logrus.Debugf("S3File: Seek %d offset %d whence", offset, whence)
-
-	var abs int64
-	switch whence {
-	case io.SeekStart:
-		abs = offset
-	case io.SeekCurrent:
-		abs = f.i + offset
-	case io.SeekEnd:
-		abs = f.size + offset
-	default:
-		return 0, errors.New("S3File: invalid whence")
-	}
-	if abs < 0 {
-		return 0, errors.New("S3File: negative position")
-	}
-	f.i = abs
-	return abs, nil
-}
-
-func (f *S3File) Reset() {
-	f.i = 0
-}
-
-func (f *S3File) Close() error {
-	f.client = nil
-	f.rcache = nil
-	return nil
-}
-
-func (f *S3File) Clone() *S3File {
-	return &S3File{
-		s3uri:  f.s3uri,
-		client: f.client,
-		i:      0,
-		size:   f.size,
-		rcache: f.rcache,
-	}
-}
-
-// WriteTo implements the io.WriterTo interface.
-// func (f *S3File) WriteTo(w io.Writer) (n int64, err error) {
-// 	logrus.Debugf("S3File: WriteTo")
-
-// 	if f.i >= f.size {
-// 		return 0, io.EOF
-// 	}
-
-// 	wa, ok := w.(io.WriterAt)
-// 	if !ok {
-// 		return 0, errors.New("S3File: writer must be io.WriterAt")
-// 	}
-
-// 	downloader := manager.NewDownloader(f.client)
-// 	n, err = downloader.Download(context.TODO(), wa, &s3.GetObjectInput{
-// 		Bucket: &f.s3uri.Bucket,
-// 		Key:    &f.s3uri.Key,
-// 		Range:  aws.String(fmt.Sprintf("bytes=%d-", f.i)),
-// 	})
-// 	f.i += n
-// 	return
-// }
-
-func NewS3File(cfg aws.Config, s3uri S3Uri) (*S3File, error) {
-	client := s3.NewFromConfig(cfg)
-	output, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
-		Bucket: &s3uri.Bucket,
-		Key:    &s3uri.Key,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return &S3File{
-		s3uri:  s3uri,
-		client: client,
-		i:      0,
-		size:   *output.ContentLength,
-		// The total cache size is `iolimits.CacheBlockCount * iolimits.BlockSize`
-		rcache: NewBlockCache(iolimits.CacheBlockCount),
-	}, nil
-}
-
-type Block struct {
-	Id  int64
-	Buf []byte
-}
-
-func (b *Block) Size() int {
-	return len(b.Buf)
-}
-
-type LRUBlockPool struct {
-	pool  *sync.Pool
-	cache *lru.Cache
-	mutex sync.Mutex
-}
-
-func NewLRUBlockPool(capacity int) *LRUBlockPool {
-	pool := &sync.Pool{
-		New: func() interface{} {
-			return &Block{
-				Id:  -1,
-				Buf: make([]byte, iolimits.BlockSize),
-			}
-		},
-	}
-	cache := lru.New(capacity)
-	cache.OnEvicted = func(k lru.Key, v interface{}) {
-		pool.Put(v)
-	}
-	return &LRUBlockPool{
-		pool:  pool,
-		cache: cache,
-	}
-}
-
-func (p *LRUBlockPool) GetBlock(id int64, blockInitFn func(*Block) error) (block *Block, err error) {
-	p.mutex.Lock()
-	defer p.mutex.Unlock()
-	val, hit := p.cache.Get(id)
-	if hit {
-		if block, ok := val.(*Block); ok {
-			return block, nil
-		} else {
-			return nil, errors.New("get an invalid block from cache")
-		}
-	} else {
-		logrus.Debugf("LRUBlockPool: miss block#%d", id)
-		if (p.cache.MaxEntries != 0) && (p.cache.Len() >= p.cache.MaxEntries) {
-			p.cache.RemoveOldest()
-		}
-		blk := p.pool.Get()
-		if block, ok := blk.(*Block); ok {
-			block.Id = id
-			err = blockInitFn(block)
-			p.cache.Add(id, block)
-			return block, err
-		} else {
-			return nil, errors.New("get an invalid block from pool")
-		}
-	}
-}
-
-type CacheMissFn func(b *Block) error
-
-type BlockCache struct {
-	pool *LRUBlockPool
-}
-
-func NewBlockCache(capacity int) *BlockCache {
-	return &BlockCache{
-		pool: NewLRUBlockPool(capacity),
-	}
-}
-
-func (c *BlockCache) Read(begin, end int64, cacheMissFn CacheMissFn) (buf []byte, err error) {
-	if begin < 0 {
-		return nil, fmt.Errorf("LRUBlockCache: negative begin")
-	}
-	if end < 0 {
-		return nil, fmt.Errorf("LRUBlockCache: negative end")
-	}
-	if begin >= end {
-		return nil, fmt.Errorf("LRUBlockCache: byte end must greater than byte begin")
-	}
-	bidBegin := begin / iolimits.BlockSize
-	bidEnd := end / iolimits.BlockSize
-	buf = make([]byte, 0)
-
-	for bid := bidBegin; bid <= bidEnd; bid++ {
-		b, e := blockAddressTranslation(begin, end, bid)
-		block, err := c.pool.GetBlock(bid, cacheMissFn)
-		if err != nil || block == nil {
-			return nil, errors.Wrapf(err, "error when get block from pool")
-		}
-		buf = append(buf, block.Buf[b:e]...)
-	}
-	return buf, nil
-}
-
-// Returns the byte range of the block at the given begin and end address
-func blockAddressTranslation(begin, end, bid int64) (b, e int64) {
-	b = max(begin, bid*iolimits.BlockSize) - bid*iolimits.BlockSize
-	e = min(end, (bid+1)*iolimits.BlockSize) - bid*iolimits.BlockSize
-	return
-}
-
-func max(a, b int64) int64 {
-	if a > b {
-		return a
-	}
-	return b
-}
-
-func min(a, b int64) int64 {
-	if a < b {
-		return a
-	}
-	return b
-}

package/lambda-src/internal/tarfile/s3file_test.go

@@ -1,135 +0,0 @@
-// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-package tarfile
-
-import (
-	"archive/tar"
-	"cdk-ecr-deployment-handler/internal/iolimits"
-	"context"
-	"fmt"
-	"io"
-	"log"
-	"testing"
-
-	"github.com/aws/aws-sdk-go-v2/config"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestNewS3File(t *testing.T) {
-	t.Skip()
-	cfg, err := config.LoadDefaultConfig(context.TODO())
-	assert.NoError(t, err)
-
-	s3uri, _ := ParseS3Uri("s3://cdk-ecr-deployment/nginx.tar")
-
-	f, err := NewS3File(cfg, *s3uri)
-	assert.NoError(t, err)
-
-	log.Printf("file size: %d", f.Size())
-
-	tr := tar.NewReader(f)
-	for {
-		hdr, err := tr.Next()
-		if err == io.EOF {
-			break // End of archive
-		}
-		if err != nil {
-			log.Fatal(err)
-		}
-		fmt.Printf("%s\n", hdr.Name)
-	}
-}
-
-func TestBlockAddressTranslation(t *testing.T) {
-	begin := int64(iolimits.BlockSize - iolimits.MegaByte)
-	end := int64(3*iolimits.BlockSize - iolimits.MegaByte)
-
-	b, e := blockAddressTranslation(begin, end, 0)
-	assert.Equal(t, begin, b)
-	assert.Equal(t, int64(iolimits.BlockSize), e)
-
-	b, e = blockAddressTranslation(begin, end, 1)
-	assert.Equal(t, int64(0), b)
-	assert.Equal(t, int64(iolimits.BlockSize), e)
-
-	b, e = blockAddressTranslation(begin, end, 2)
-	assert.Equal(t, int64(0), b)
-	assert.Equal(t, int64(iolimits.BlockSize-iolimits.MegaByte), e)
-}
-
-func TestBlockCache(t *testing.T) {
-	n := 0
-	cache := NewBlockCache(1)
-	cacheMissFn := func(block *Block) error {
-		n++
-		copy(block.Buf, magic(block.Id))
-		return nil
-	}
-
-	// read 0-3 bytes of block0
-	buf, err := cache.Read(0, 3, cacheMissFn)
-	assert.NoError(t, err)
-	assert.Equal(t, 1, n)
-	assert.Equal(t, magic(0), buf)
-
-	// read 0-3 bytes of block0's cache
-	buf, err = cache.Read(0, 3, cacheMissFn)
-	assert.NoError(t, err)
-	assert.Equal(t, 1, n)
-	assert.Equal(t, magic(0), buf)
-
-	// read 0-3 bytes of block1
-	buf, err = cache.Read(iolimits.BlockSize, iolimits.BlockSize+3, cacheMissFn)
-	assert.NoError(t, err)
-	assert.Equal(t, 2, n)
-	assert.Equal(t, magic(1), buf)
-
-	// read whole block1 and 0-3 bytes of block2
-	buf, err = cache.Read(0, iolimits.BlockSize+3, cacheMissFn)
-	assert.NoError(t, err)
-	assert.Equal(t, 4, n)
-	assert.Equal(t, append(mkblk(magic(0)), magic(1)...), buf)
-}
-
-func TestLRUBlockPool(t *testing.T) {
-	n := 0
-	pool := NewLRUBlockPool(1)
-	blockInitFn := func(block *Block) error {
-		n++
-		return nil
-	}
-
-	block, err := pool.GetBlock(0, blockInitFn)
-	assert.NoError(t, err)
-	assert.Equal(t, 1, n)
-	assert.Equal(t, int64(0), block.Id)
-	assert.Equal(t, iolimits.BlockSize, block.Size())
-	block.Buf[0] = byte('A')
-
-	block, err = pool.GetBlock(1, blockInitFn)
-	assert.NoError(t, err)
-	assert.Equal(t, 2, n)
-	assert.Equal(t, int64(1), block.Id)
-	assert.Equal(t, iolimits.BlockSize, block.Size())
-	assert.Equal(t, byte('A'), block.Buf[0])
-	block.Buf[0] = byte('B')
-
-	block, err = pool.GetBlock(1, blockInitFn)
-	assert.NoError(t, err)
-	assert.Equal(t, 2, n)
-	assert.Equal(t, int64(1), block.Id)
-	assert.Equal(t, iolimits.BlockSize, block.Size())
-	assert.Equal(t, byte('B'), block.Buf[0])
-}
-
-// Create magic bytes based on seed: [seed-1, seed, seed+1]
-func magic(seed int64) []byte {
-	return []byte{byte(seed - 1), byte(seed), byte(seed + 1)}
-}
-
-func mkblk(init []byte) []byte {
-	block := make([]byte, iolimits.BlockSize)
-	copy(block[0:len(init)], init)
-	return block
-}