feedx 0.12.7 → 0.14.0

data/producer.go

@@ -2,8 +2,6 @@ package feedx
 
 import (
 	"context"
-	"sync/atomic"
-	"time"
 
 	"github.com/bsm/bfs"
 )
@@ -11,146 +9,66 @@ import (
 // ProduceFunc is a callback which is run by the producer on every iteration.
 type ProduceFunc func(*Writer) error
 
-// ProducerOptions configure the producer instance.
-type ProducerOptions struct {
-	WriterOptions
-
-	// The interval used by producer to initiate a cycle.
-	// Default: 1m
-	Interval time.Duration
-
-	// LastModCheck this function will be called before each push attempt
-	// to dynamically determine the last modified time.
-	LastModCheck func(context.Context) (time.Time, error)
-
-	// AfterPush callbacks are triggered after each push cycle, receiving
-	// the push state and error (if occurred).
-	AfterPush func(*ProducerPush, error)
-}
-
-func (o *ProducerOptions) norm(name string) {
-	o.WriterOptions.norm(name)
-	if o.Interval <= 0 {
-		o.Interval = time.Minute
-	}
-}
-
-// ProducerPush contains the state of the last push.
-type ProducerPush struct {
-	// Producer exposes the current producer state.
-	*Producer
-	// Updated indicates is the push resulted in an update.
-	Updated bool
-}
-
-// Producer (continously) produces a feed.
+// Producer instances push data feeds to remote locations.
 type Producer struct {
 	remote    *bfs.Object
 	ownRemote bool
-
-	opt  ProducerOptions
-	ctx  context.Context
-	stop context.CancelFunc
-	pfn  ProduceFunc
-
-	numWritten, lastPush, lastMod int64
 }
 
 // NewProducer inits a new feed producer.
-func NewProducer(ctx context.Context, remoteURL string, opt *ProducerOptions, pfn ProduceFunc) (*Producer, error) {
+func NewProducer(ctx context.Context, remoteURL string) (*Producer, error) {
 	remote, err := bfs.NewObject(ctx, remoteURL)
 	if err != nil {
 		return nil, err
 	}
 
-	p, err := NewProducerForRemote(ctx, remote, opt, pfn)
-	if err != nil {
-		_ = remote.Close()
-		return nil, err
-	}
-	p.ownRemote = true
-	return p, nil
+	pcr := NewProducerForRemote(remote)
+	pcr.ownRemote = true
+	return pcr, nil
 }
 
 // NewProducerForRemote starts a new feed producer with a remote.
-func NewProducerForRemote(ctx context.Context, remote *bfs.Object, opt *ProducerOptions, pfn ProduceFunc) (*Producer, error) {
-	var o ProducerOptions
-	if opt != nil {
-		o = *opt
-	}
-	o.norm(remote.Name())
-
-	ctx, stop := context.WithCancel(ctx)
-	p := &Producer{
-		remote: remote,
-		opt:    o,
-		pfn:    pfn,
-		ctx:    ctx,
-		stop:   stop,
-	}
-
-	// run initial push
-	if _, err := p.push(); err != nil {
-		_ = p.Close()
-		return nil, err
-	}
-
-	// start continuous loop
-	go p.loop()
-
-	return p, nil
-}
-
-// LastPush returns time of last push attempt.
-func (p *Producer) LastPush() time.Time {
-	return timestamp(atomic.LoadInt64(&p.lastPush)).Time()
-}
-
-// LastModified returns time at which the remote feed was last modified.
-func (p *Producer) LastModified() time.Time {
-	return timestamp(atomic.LoadInt64(&p.lastMod)).Time()
-}
-
-// NumWritten returns the number of values produced during the last push.
-func (p *Producer) NumWritten() int {
-	return int(atomic.LoadInt64(&p.numWritten))
+func NewProducerForRemote(remote *bfs.Object) *Producer {
+	return &Producer{remote: remote}
 }
 
 // Close stops the producer.
 func (p *Producer) Close() error {
-	p.stop()
-	if p.ownRemote {
-		return p.remote.Close()
+	if p.ownRemote && p.remote != nil {
+		err := p.remote.Close()
+		p.remote = nil
+		return err
 	}
 	return nil
 }
 
-func (p *Producer) push() (*ProducerPush, error) {
-	start := time.Now()
-	atomic.StoreInt64(&p.lastPush, timestampFromTime(start).Millis())
-
-	// setup write options
-	wopt := p.opt.WriterOptions
-	wopt.LastMod = start
-	if p.opt.LastModCheck != nil {
-		modTime, err := p.opt.LastModCheck(p.ctx)
-		if err != nil {
-			return nil, err
-		}
-		wopt.LastMod = modTime
-	}
+func (p *Producer) Produce(ctx context.Context, version int64, opt *WriterOptions, pfn ProduceFunc) (*Status, error) {
+	status := Status{LocalVersion: version}
 
-	// retrieve original last modified time, skip if not modified
-	if rts, err := remoteLastModified(p.ctx, p.remote); err != nil {
+	// retrieve previous remote version
+	remoteVersion, err := fetchRemoteVersion(ctx, p.remote)
+	if err != nil {
 		return nil, err
-	} else if rts == timestampFromTime(wopt.LastMod) {
-		return &ProducerPush{Producer: p}, nil
 	}
+	status.RemoteVersion = remoteVersion
 
-	writer := NewWriter(p.ctx, p.remote, &wopt)
+	// skip if not modified
+	if skipSync(version, remoteVersion) {
+		status.Skipped = true
+		return &status, nil
+	}
+
+	// set version for writer
+	if opt == nil {
+		opt = new(WriterOptions)
+	}
+	opt.Version = version
+
+	// init writer and perform
+	writer := NewWriter(ctx, p.remote, opt)
 	defer writer.Discard()
 
-	if err := p.pfn(writer); err != nil {
+	if err := pfn(writer); err != nil {
 		return nil, err
 	}
 
@@ -158,27 +76,6 @@ func (p *Producer) push() (*ProducerPush, error) {
 		return nil, err
 	}
 
-	atomic.StoreInt64(&p.numWritten, int64(writer.NumWritten()))
-	atomic.StoreInt64(&p.lastMod, timestampFromTime(wopt.LastMod).Millis())
-	return &ProducerPush{
-		Producer: p,
-		Updated:  true,
-	}, nil
-}
-
-func (p *Producer) loop() {
-	ticker := time.NewTicker(p.opt.Interval)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-p.ctx.Done():
-			return
-		case <-ticker.C:
-			state, err := p.push()
-			if p.opt.AfterPush != nil {
-				p.opt.AfterPush(state, err)
-			}
-		}
-	}
+	status.NumItems = writer.NumWritten()
+	return &status, nil
 }

data/producer_test.go

@@ -1,79 +1,65 @@
 package feedx_test
 
 import (
-	"context"
-	"sync/atomic"
-	"time"
+	"reflect"
+	"testing"
 
 	"github.com/bsm/bfs"
 	"github.com/bsm/feedx"
-	. "github.com/bsm/ginkgo"
-	. "github.com/bsm/gomega"
 )
 
-var _ = Describe("Producer", func() {
-	var subject *feedx.Producer
-	var obj *bfs.Object
-	var numRuns uint32
-	var ctx = context.Background()
+func TestProducer(t *testing.T) {
+	obj := bfs.NewInMemObject("path/to/file.json")
+	defer obj.Close()
 
-	setup := func(o *feedx.ProducerOptions) {
-		var err error
-		subject, err = feedx.NewProducerForRemote(ctx, obj, o, func(w *feedx.Writer) error {
-			atomic.AddUint32(&numRuns, 1)
+	pcr := feedx.NewProducerForRemote(obj)
+	defer pcr.Close()
 
-			for i := 0; i < 10; i++ {
-				if err := w.Encode(seed()); err != nil {
-					return err
-				}
-			}
-			return nil
-		})
-		Expect(err).NotTo(HaveOccurred())
-	}
-
-	BeforeEach(func() {
-		atomic.StoreUint32(&numRuns, 0)
-		obj = bfs.NewInMemObject("path/to/file.jsonz")
+	// first attempt
+	testProduce(t, pcr, 101, &feedx.Status{
+		LocalVersion: 101,
+		NumItems:     10,
 	})
 
-	AfterEach(func() {
-		if subject != nil {
-			Expect(subject.Close()).To(Succeed())
-		}
+	// second attempt
+	testProduce(t, pcr, 101, &feedx.Status{
+		LocalVersion:  101,
+		RemoteVersion: 101,
+		Skipped:       true,
 	})
 
-	It("produces", func() {
-		setup(nil)
-
-		Expect(subject.LastPush()).To(BeTemporally("~", time.Now(), time.Second))
-		Expect(subject.LastModified()).To(BeTemporally("~", time.Now(), time.Second))
-		Expect(subject.NumWritten()).To(Equal(10))
-		Expect(subject.Close()).To(Succeed())
-
-		info, err := obj.Head(ctx)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(info.Size).To(BeNumerically("~", 75, 10))
+	// updated version
+	testProduce(t, pcr, 134, &feedx.Status{
+		LocalVersion:  134,
+		RemoteVersion: 101,
+		NumItems:      13,
 	})
 
-	It("produces with custom last-mod check", func() {
-		setup(&feedx.ProducerOptions{
-			Interval:     50 * time.Millisecond,
-			LastModCheck: func(_ context.Context) (time.Time, error) { return time.Unix(1515151515, 987654321), nil },
-		})
-
-		firstPush := subject.LastPush()
-		Expect(firstPush).To(BeTemporally("~", time.Now(), time.Second))
-		Expect(subject.LastModified()).To(Equal(time.Unix(1515151515, 987000000)))
-		Expect(subject.NumWritten()).To(Equal(10))
-		Expect(atomic.LoadUint32(&numRuns)).To(Equal(uint32(1)))
+	meta, err := obj.Head(t.Context())
+	if err != nil {
+		t.Fatal("unexpected error", err)
+	}
+	if exp := (bfs.Metadata{"X-Feedx-Version": "134"}); !reflect.DeepEqual(exp, meta.Metadata) {
+		t.Errorf("expected %#v, got %#v", exp, meta)
+	}
+}
 
-		info, err := obj.Head(ctx)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(info.Size).To(BeNumerically("~", 75, 10))
-		Expect(info.Metadata).To(HaveKeyWithValue("X-Feedx-Last-Modified", "1515151515987"))
+func testProduce(t *testing.T, pcr *feedx.Producer, version int64, exp *feedx.Status) {
+	t.Helper()
 
-		Eventually(func() bool { return subject.LastPush().After(firstPush) }).Should(BeTrue())
-		Expect(atomic.LoadUint32(&numRuns)).To(Equal(uint32(1)))
+	status, err := pcr.Produce(t.Context(), version, nil, func(w *feedx.Writer) error {
+		for i := int64(0); i < version/10; i++ {
+			if err := w.Encode(seed()); err != nil {
+				return err
+			}
+		}
+		return nil
 	})
-})
+	if err != nil {
+		t.Fatal("unexpected error", err)
+	}
+
+	if !reflect.DeepEqual(exp, status) {
+		t.Errorf("expected %#v, got %#v", exp, status)
+	}
+}

data/reader.go

@@ -2,8 +2,8 @@ package feedx
 
 import (
 	"context"
+	"errors"
 	"io"
-	"time"
 
 	"github.com/bsm/bfs"
 )
@@ -30,42 +30,159 @@ func (o *ReaderOptions) norm(name string) {
 
 // Reader reads data from a remote feed.
 type Reader struct {
-	remote *bfs.Object
-	opt    ReaderOptions
-	ctx    context.Context
-	num    int
+	ctx context.Context
+	opt *ReaderOptions
 
-	br io.ReadCloser // bfs reader
-	cr io.ReadCloser // compression reader
-	fd FormatDecoder
+	remotes    []*bfs.Object
+	ownRemotes bool
+
+	cur *streamReader
+	pos int
+
+	num int64
 }
 
 // NewReader inits a new reader.
 func NewReader(ctx context.Context, remote *bfs.Object, opt *ReaderOptions) (*Reader, error) {
-	var o ReaderOptions
-	if opt != nil {
-		o = *opt
-	}
-	o.norm(remote.Name())
+	return MultiReader(ctx, []*bfs.Object{remote}, opt), nil
+}
 
+// MultiReader inits a new reader for multiple remotes.  Remotes are read sequentially as if concatenated.
+// Once all remotes are fully read, Read will return EOF.
+func MultiReader(ctx context.Context, remotes []*bfs.Object, opt *ReaderOptions) *Reader {
 	return &Reader{
-		remote: remote,
-		opt:    o,
-		ctx:    ctx,
-	}, nil
+		remotes: remotes,
+		opt:     opt,
+		ctx:     ctx,
+	}
 }
 
 // Read reads raw bytes from the feed.
+// At end of feed, Read returns 0, io.EOF.
 func (r *Reader) Read(p []byte) (int, error) {
+	if !r.ensureCurrent() {
+		return 0, io.EOF
+	}
+
+	n, err := r.cur.Read(p)
+	if errors.Is(err, io.EOF) {
+		if more, err := r.nextRemote(); err != nil {
+			return n, err
+		} else if more {
+			return n, nil // dont return EOF until all remotes read
+		}
+	}
+	return n, err
+}
+
+// Decode decodes the next formatted value from the feed.
+// At end of feed, Read returns io.EOF.
+func (r *Reader) Decode(v interface{}) error {
+	if !r.ensureCurrent() {
+		return io.EOF
+	}
+
+	err := r.cur.Decode(v)
+	if errors.Is(err, io.EOF) {
+		if more, err := r.nextRemote(); err != nil {
+			return err
+		} else if more {
+			return r.Decode(v) // start decoding from next remote
+		}
+	} else if err == nil {
+		r.num++
+	}
+	return err
+}
+
+// NumRead returns the number of read values.
+func (r *Reader) NumRead() int64 {
+	return r.num
+}
+
+// Version returns the version of the remote feed.
+func (r *Reader) Version() (int64, error) {
+	var max int64
+	for _, remote := range r.remotes {
+		v, err := fetchRemoteVersion(r.ctx, remote)
+		if err != nil {
+			return 0, err
+		} else if v > max {
+			max = v
+		}
+	}
+
+	return max, nil
+}
+
+// Close closes the reader.
+func (r *Reader) Close() (err error) {
+	if r.cur != nil {
+		err = r.cur.Close()
+	}
+	if r.ownRemotes {
+		for _, remote := range r.remotes {
+			if e := remote.Close(); e != nil {
+				err = errors.Join(err, e)
+			}
+		}
+	}
+	return
+}
+
+func (r *Reader) ensureCurrent() bool {
+	if r.pos >= len(r.remotes) {
+		return false
+	}
+
+	if r.cur == nil {
+		remote := r.remotes[r.pos]
+
+		var o ReaderOptions
+		if r.opt != nil {
+			o = *r.opt
+		}
+		o.norm(remote.Name())
+
+		r.cur = &streamReader{
+			remote: remote,
+			opt:    o,
+			ctx:    r.ctx,
+		}
+	}
+	return true
+}
+
+func (r *Reader) nextRemote() (bool, error) {
+	if err := r.cur.Close(); err != nil {
+		return false, err
+	}
+	// unset current, increment cursor
+	r.cur = nil
+	r.pos++
+	return r.pos < len(r.remotes), nil
+}
+
+type streamReader struct {
+	remote *bfs.Object
+	opt    ReaderOptions
+	ctx    context.Context
+
+	br io.ReadCloser // bfs reader
+	cr io.ReadCloser // compression reader
+	fd FormatDecoder
+}
+
+// Read reads raw bytes from the feed.
+func (r *streamReader) Read(p []byte) (int, error) {
 	if err := r.ensureOpen(); err != nil {
 		return 0, err
 	}
-
 	return r.cr.Read(p)
 }
 
 // Decode decodes the next formatted value from the feed.
-func (r *Reader) Decode(v interface{}) error {
+func (r *streamReader) Decode(v interface{}) error {
 	if err := r.ensureOpen(); err != nil {
 		return err
 	}
@@ -78,47 +195,31 @@ func (r *Reader) Decode(v interface{}) error {
 		r.fd = fd
 	}
 
-	if err := r.fd.Decode(v); err != nil {
-		return err
-	}
-
-	r.num++
-	return nil
-}
-
-// NumRead returns the number of read values.
-func (r *Reader) NumRead() int {
-	return r.num
-}
-
-// LastModified returns the last modified time of the remote feed.
-func (r *Reader) LastModified() (time.Time, error) {
-	lastMod, err := remoteLastModified(r.ctx, r.remote)
-	return lastMod.Time(), err
+	return r.fd.Decode(v)
 }
 
 // Close closes the reader.
-func (r *Reader) Close() error {
+func (r *streamReader) Close() error {
 	var err error
 	if r.fd != nil {
 		if e := r.fd.Close(); e != nil {
-			err = e
+			err = errors.Join(err, e)
 		}
 	}
 	if r.cr != nil {
 		if e := r.cr.Close(); e != nil {
-			err = e
+			err = errors.Join(err, e)
 		}
 	}
 	if r.br != nil {
 		if e := r.br.Close(); e != nil {
-			err = e
+			err = errors.Join(err, e)
 		}
 	}
 	return err
 }
 
-func (r *Reader) ensureOpen() error {
+func (r *streamReader) ensureOpen() error {
 	if r.br == nil {
 		br, err := r.remote.Open(r.ctx)
 		if err != nil {