feedx 0.12.7 → 0.15.0

data/reader_test.go

@@ -1,56 +1,107 @@
 package feedx_test
 
 import (
-	"context"
 	"io"
-	"io/ioutil"
-	"time"
+	"reflect"
+	"testing"
 
 	"github.com/bsm/bfs"
 	"github.com/bsm/feedx"
 	"github.com/bsm/feedx/internal/testdata"
-	. "github.com/bsm/ginkgo"
-	. "github.com/bsm/gomega"
 )
 
-var _ = Describe("Reader", func() {
-	var subject *feedx.Reader
-	var obj *bfs.Object
-	var ctx = context.Background()
+func TestReader(t *testing.T) {
+	t.Run("reads", func(t *testing.T) {
+		r := fixReader(t)
 
-	BeforeEach(func() {
-		obj = bfs.NewInMemObject("path/to/file.json")
-		Expect(writeMulti(obj, 3, time.Time{})).To(Succeed())
+		if data, err := io.ReadAll(r); err != nil {
+			t.Fatal("unexpected error", err)
+		} else if exp, got := 111, len(data); exp != got {
+			t.Errorf("expected %v, got %v", exp, got)
+		} else if exp, got := int64(0), r.NumRead(); exp != got {
+			t.Errorf("expected %v, got %v", exp, got)
+		}
+	})
 
-		var err error
-		subject, err = feedx.NewReader(ctx, obj, nil)
-		Expect(err).NotTo(HaveOccurred())
+	t.Run("decodes", func(t *testing.T) {
+		r := fixReader(t)
+		msgs := drainReader(t, r)
+		if exp := seedN(3); !reflect.DeepEqual(exp, msgs) {
+			t.Errorf("expected %#v, got %#v", exp, msgs)
+		}
+		if exp, got := int64(3), r.NumRead(); exp != got {
+			t.Errorf("expected %v, got %v", exp, got)
+		}
 	})
+}
+
+func fixReader(t *testing.T) *feedx.Reader {
+	t.Helper()
 
-	AfterEach(func() {
-		Expect(subject.Close()).To(Succeed())
+	obj := bfs.NewInMemObject("path/to/file.jsonz")
+	if err := writeN(obj, 3, 0); err != nil {
+		t.Fatal("unexpected error", err)
+	}
+
+	r, err := feedx.NewReader(t.Context(), obj, nil)
+	if err != nil {
+		t.Fatal("unexpected error", err)
+	}
+
+	t.Cleanup(func() {
+		_ = r.Close()
 	})
 
-	It("reads", func() {
-		data, err := ioutil.ReadAll(subject)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(len(data)).To(BeNumerically("~", 110, 20))
-		Expect(subject.NumRead()).To(Equal(0))
+	return r
+}
+
+func TestMultiReader(t *testing.T) {
+	t.Run("reads", func(t *testing.T) {
+		r := fixMultiReader(t)
+
+		if data, err := io.ReadAll(r); err != nil {
+			t.Fatal("unexpected error", err)
+		} else if exp, got := 222, len(data); exp != got {
+			t.Errorf("expected %v, got %v", exp, got)
+		} else if exp, got := int64(0), r.NumRead(); exp != got {
+			t.Errorf("expected %v, got %v", exp, got)
+		}
 	})
 
-	It("decodes", func() {
-		var msgs []*testdata.MockMessage
-		for {
-			var msg testdata.MockMessage
-			err := subject.Decode(&msg)
-			if err == io.EOF {
-				break
-			}
-			Expect(err).NotTo(HaveOccurred())
-			msgs = append(msgs, &msg)
+	t.Run("decodes", func(t *testing.T) {
+		r := fixMultiReader(t)
+		msgs := drainReader(t, r)
+		if exp := seedN(6); !reflect.DeepEqual(exp, msgs) {
+			t.Errorf("expected %#v, got %#v", exp, msgs)
+		}
+		if exp, got := int64(6), r.NumRead(); exp != got {
+			t.Errorf("expected %v, got %v", exp, got)
 		}
+	})
+}
+
+func fixMultiReader(t *testing.T) *feedx.Reader {
+	t.Helper()
 
-		Expect(msgs).To(ConsistOf(seed(), seed(), seed()))
-		Expect(subject.NumRead()).To(Equal(3))
+	obj := bfs.NewInMemObject("path/to/file.jsonz")
+	if err := writeN(obj, 3, 0); err != nil {
+		t.Fatal("unexpected error", err)
+	}
+
+	r := feedx.MultiReader(t.Context(), []*bfs.Object{obj, obj}, nil)
+	t.Cleanup(func() {
+		_ = r.Close()
 	})
-})
+
+	return r
+}
+
+func drainReader(t *testing.T, r interface{ Decode(any) error }) []*testdata.MockMessage {
+	t.Helper()
+
+	msgs, err := readMessages(r)
+	if err != nil {
+		t.Fatal("unexpected error", err)
+	}
+	return msgs
+}

data/scheduler.go

@@ -0,0 +1,179 @@
+package feedx
+
+import (
+	"context"
+	"sync"
+	"time"
+)
+
+// BeforeHook callbacks are run before jobs are started. It receives the local
+// version before sync as an argument and may return false to abort the cycle.
+type BeforeHook func(version int64) bool
+
+// AfterHook callbacks are run after jobs have finished.
+type AfterHook func(*Status, error)
+
+// VersionCheck callbacks return the latest local version.
+type VersionCheck func(context.Context) (int64, error)
+
+// Scheduler runs cronjobs in regular intervals.
+type Scheduler struct {
+	ctx      context.Context
+	interval time.Duration
+
+	readerOpt    *ReaderOptions
+	writerOpt    *WriterOptions
+	versionCheck VersionCheck
+
+	// hooks
+	beforeHooks []BeforeHook
+	afterHooks  []AfterHook
+}
+
+// Every creates a scheduler.
+func Every(interval time.Duration) *Scheduler {
+	return &Scheduler{ctx: context.Background(), interval: interval}
+}
+
+// WithContext sets a custom context for the run.
+func (s *Scheduler) WithContext(ctx context.Context) *Scheduler {
+	s.ctx = ctx
+	return s
+}
+
+// BeforeSync adds custom before hooks.
+func (s *Scheduler) BeforeSync(hooks ...BeforeHook) *Scheduler {
+	s.beforeHooks = append(s.beforeHooks, hooks...)
+	return s
+}
+
+// AfterSync adds before hooks.
+func (s *Scheduler) AfterSync(hooks ...AfterHook) *Scheduler {
+	s.afterHooks = append(s.afterHooks, hooks...)
+	return s
+}
+
+// WithReaderOptions sets custom reader options for consumers.
+func (s *Scheduler) WithReaderOptions(opt *ReaderOptions) *Scheduler {
+	s.readerOpt = opt
+	return s
+}
+
+// Consume starts a consumer job.
+func (s *Scheduler) Consume(csm Consumer, cfn ConsumeFunc) (*CronJob, error) {
+	return newCronJob(s.ctx, s.interval, func(ctx context.Context) error {
+		version := csm.Version()
+		if !s.runBeforeHooks(version) {
+			return nil
+		}
+
+		status, err := csm.Consume(ctx, s.readerOpt, cfn)
+		s.runAfterHooks(status, err)
+		return err
+	})
+}
+
+// WithWriterOptions sets custom writer options for producers.
+func (s *Scheduler) WithWriterOptions(opt *WriterOptions) *Scheduler {
+	s.writerOpt = opt
+	return s
+}
+
+// WithVersionCheck sets a custom version check for producers.
+func (s *Scheduler) WithVersionCheck(fn VersionCheck) *Scheduler {
+	s.versionCheck = fn
+	return s
+}
+
+// Produce starts a producer job.
+func (s *Scheduler) Produce(pcr *Producer, pfn ProduceFunc) (*CronJob, error) {
+	return s.produce(func(ctx context.Context, version int64) (*Status, error) {
+		return pcr.Produce(ctx, version, s.writerOpt, pfn)
+	})
+}
+
+// ProduceIncrementally starts an incremental producer job.
+func (s *Scheduler) ProduceIncrementally(pcr *IncrementalProducer, pfn IncrementalProduceFunc) (*CronJob, error) {
+	return s.produce(func(ctx context.Context, version int64) (*Status, error) {
+		return pcr.Produce(ctx, version, s.writerOpt, pfn)
+	})
+}
+
+func (s *Scheduler) produce(fn func(context.Context, int64) (*Status, error)) (*CronJob, error) {
+	return newCronJob(s.ctx, s.interval, func(ctx context.Context) error {
+		var version int64
+		if s.versionCheck != nil {
+			latest, err := s.versionCheck(s.ctx)
+			if err != nil {
+				s.runAfterHooks(nil, err)
+				return err
+			}
+			version = latest
+		}
+
+		if !s.runBeforeHooks(version) {
+			return nil
+		}
+
+		status, err := fn(ctx, version)
+		s.runAfterHooks(status, err)
+		return err
+	})
+}
+
+func (s *Scheduler) runBeforeHooks(version int64) bool {
+	for _, hook := range s.beforeHooks {
+		if !hook(version) {
+			return false
+		}
+	}
+	return true
+}
+
+func (s *Scheduler) runAfterHooks(status *Status, err error) {
+	for _, hook := range s.afterHooks {
+		hook(status, err)
+	}
+}
+
+// CronJob runs in regular intervals until it's stopped.
+type CronJob struct {
+	cancel   context.CancelFunc
+	interval time.Duration
+	perform  func(context.Context) error
+	wait     sync.WaitGroup
+}
+
+func newCronJob(ctx context.Context, interval time.Duration, perform func(context.Context) error) (*CronJob, error) {
+	if err := perform(ctx); err != nil {
+		return nil, err
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+	job := &CronJob{cancel: cancel, interval: interval, perform: perform}
+	go job.loop(ctx)
+	return job, nil
+}
+
+// Stop stops the job and waits until it is complete.
+func (j *CronJob) Stop() {
+	j.cancel()
+	j.wait.Wait()
+}
+
+func (j *CronJob) loop(ctx context.Context) {
+	j.wait.Add(1)
+	defer j.wait.Done()
+
+	ticker := time.NewTicker(j.interval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			_ = j.perform(ctx)
+		}
+	}
+}

data/scheduler_test.go

@@ -0,0 +1,171 @@
+package feedx_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/bsm/bfs"
+	"github.com/bsm/feedx"
+)
+
+func TestScheduler(t *testing.T) {
+	beforeCallbacks := new(atomic.Int32)
+	afterCallbacks := new(atomic.Int32)
+	numCycles := new(atomic.Int32)
+	numErrors := new(atomic.Int32)
+
+	resetCounters := func() {
+		beforeCallbacks.Store(0)
+		afterCallbacks.Store(0)
+		numCycles.Store(0)
+		numErrors.Store(0)
+	}
+
+	obj := bfs.NewInMemObject("file.json")
+	defer obj.Close()
+
+	t.Run("produce", func(t *testing.T) {
+		resetCounters()
+
+		pcr := feedx.NewProducerForRemote(obj)
+		defer pcr.Close()
+
+		job, err := feedx.Every(time.Millisecond).
+			BeforeSync(func(_ int64) bool {
+				beforeCallbacks.Add(1)
+				return true
+			}).
+			AfterSync(func(_ *feedx.Status, err error) {
+				afterCallbacks.Add(1)
+
+				if err != nil {
+					numErrors.Add(1)
+				}
+			}).
+			WithVersionCheck(func(_ context.Context) (int64, error) {
+				return 101, nil
+			}).
+			Produce(pcr, func(w *feedx.Writer) error {
+				if numCycles.Add(1)%2 == 0 {
+					return fmt.Errorf("failed!")
+				}
+				return nil
+			})
+		if err != nil {
+			t.Fatal("unexpected error", err)
+		}
+
+		time.Sleep(5 * time.Millisecond)
+		job.Stop()
+		time.Sleep(2 * time.Millisecond)
+
+		ranTimes := numCycles.Load()
+		if min, got := 4, int(ranTimes); got <= min {
+			t.Errorf("expected %d >= %d", got, min)
+		}
+		if exp, got := ranTimes, beforeCallbacks.Load(); exp != got {
+			t.Errorf("expected %d, got %d", exp, got)
+		}
+		if exp, got := ranTimes, afterCallbacks.Load(); exp != got {
+			t.Errorf("expected %d, got %d", exp, got)
+		}
+		if exp, got := ranTimes/2, numErrors.Load(); exp != got {
+			t.Errorf("expected %d, got %d", exp, got)
+		}
+
+		// wait a little longer, make sure job was stopped
+		time.Sleep(2 * time.Millisecond)
+		if exp, got := ranTimes, numCycles.Load(); exp != got {
+			t.Errorf("expected %d, got %d", exp, got)
+		}
+	})
+
+	t.Run("produce may fail", func(t *testing.T) {
+		resetCounters()
+
+		pcr := feedx.NewProducerForRemote(obj)
+		defer pcr.Close()
+
+		exp := fmt.Errorf("failed!")
+		_, err := feedx.Every(time.Millisecond).
+			Produce(pcr, func(w *feedx.Writer) error {
+				return exp
+			})
+		if !errors.Is(err, exp) {
+			t.Errorf("expected %v, got %v", exp, err)
+		}
+	})
+
+	t.Run("consume", func(t *testing.T) {
+		resetCounters()
+
+		csm := feedx.NewConsumerForRemote(obj)
+		defer csm.Close()
+
+		job, err := feedx.Every(time.Millisecond).
+			BeforeSync(func(_ int64) bool {
+				beforeCallbacks.Add(1)
+				return true
+			}).
+			AfterSync(func(_ *feedx.Status, err error) {
+				afterCallbacks.Add(1)
+
+				if err != nil {
+					numErrors.Add(1)
+				}
+			}).
+			Consume(csm, func(r *feedx.Reader) error {
+				if numCycles.Add(1)%2 == 0 {
+					return fmt.Errorf("failed!")
+				}
+				return nil
+			})
+		if err != nil {
+			t.Fatal("unexpected error", err)
+		}
+
+		time.Sleep(5 * time.Millisecond)
+		job.Stop()
+		time.Sleep(2 * time.Millisecond)
+
+		ranTimes := numCycles.Load()
+		if min, got := 4, int(ranTimes); got <= min {
+			t.Errorf("expected %d >= %d", got, min)
+		}
+		if exp, got := ranTimes, beforeCallbacks.Load(); exp != got {
+			t.Errorf("expected %d, got %d", exp, got)
+		}
+		if exp, got := ranTimes, afterCallbacks.Load(); exp != got {
+			t.Errorf("expected %d, got %d", exp, got)
+		}
+		if exp, got := ranTimes/2, numErrors.Load(); exp != got {
+			t.Errorf("expected %d, got %d", exp, got)
+		}
+
+		// wait a little longer, make sure job was stopped
+		time.Sleep(2 * time.Millisecond)
+		if exp, got := ranTimes, numCycles.Load(); exp != got {
+			t.Errorf("expected %d, got %d", exp, got)
+		}
+	})
+
+	t.Run("consume may fail", func(t *testing.T) {
+		resetCounters()
+
+		csm := feedx.NewConsumerForRemote(obj)
+		defer csm.Close()
+
+		exp := fmt.Errorf("failed!")
+		_, err := feedx.Every(time.Millisecond).
+			Consume(csm, func(r *feedx.Reader) error {
+				return exp
+			})
+		if !errors.Is(err, exp) {
+			t.Errorf("expected %v, got %v", exp, err)
+		}
+	})
+}

data/writer.go

@@ -3,8 +3,9 @@ package feedx
 import (
 	"bufio"
 	"context"
+	"errors"
 	"io"
-	"time"
+	"strconv"
 
 	"github.com/bsm/bfs"
 )
@@ -19,9 +20,9 @@ type WriterOptions struct {
 	// Default: auto-detected from URL path.
 	Compression Compression
 
-	// Provides an optional last modified timestamp which is stored with the remote metadata.
-	// Default: time.Time{}.
-	LastMod time.Time
+	// Provides an optional version which is stored with the remote metadata.
+	// Default: 0
+	Version int64
 }
 
 func (o *WriterOptions) norm(name string) {
@@ -38,7 +39,7 @@ type Writer struct {
 	ctx    context.Context
 	remote *bfs.Object
 	opt    WriterOptions
-	num    int
+	num    int64
 
 	bw bfs.Writer
 	cw io.WriteCloser // compression writer
@@ -100,7 +101,7 @@ func (w *Writer) Encode(v interface{}) error {
 }
 
 // NumWritten returns the number of written values.
-func (w *Writer) NumWritten() int {
+func (w *Writer) NumWritten() int64 {
 	return w.num
 }
 
@@ -109,7 +110,7 @@ func (w *Writer) Discard() error {
 	err := w.close()
 	if w.bw != nil {
 		if e := w.bw.Discard(); e != nil {
-			err = e
+			err = errors.Join(err, e)
 		}
 	}
 	return err
@@ -120,7 +121,7 @@ func (w *Writer) Commit() error {
 	err := w.close()
 	if w.bw != nil {
 		if e := w.bw.Commit(); e != nil {
-			err = e
+			err = errors.Join(err, e)
 		}
 	}
 	return err
@@ -129,17 +130,17 @@ func (w *Writer) Commit() error {
 func (w *Writer) close() (err error) {
 	if w.fe != nil {
 		if e := w.fe.Close(); e != nil {
-			err = e
+			err = errors.Join(err, e)
 		}
 	}
 	if w.ww != nil {
 		if e := w.ww.Flush(); e != nil {
-			err = e
+			err = errors.Join(err, e)
 		}
 	}
 	if w.cw != nil {
 		if e := w.cw.Close(); e != nil {
-			err = e
+			err = errors.Join(err, e)
 		}
 	}
 	return err
@@ -147,9 +148,8 @@ func (w *Writer) close() (err error) {
 
 func (w *Writer) ensureCreated() error {
 	if w.bw == nil {
-		ts := timestampFromTime(w.opt.LastMod)
 		bw, err := w.remote.Create(w.ctx, &bfs.WriteOptions{
-			Metadata: bfs.Metadata{metaLastModified: ts.String()},
+			Metadata: bfs.Metadata{metaVersion: strconv.FormatInt(w.opt.Version, 10)},
 		})
 		if err != nil {
 			return err