feedx 0.12.7 → 0.14.0

data/consumer.go

@@ -2,213 +2,157 @@ package feedx
 
 import (
 	"context"
+	"errors"
 	"sync/atomic"
-	"time"
 
 	"github.com/bsm/bfs"
 )
 
-// ConsumerOptions configure the consumer instance.
-type ConsumerOptions struct {
-	ReaderOptions
-
-	// The interval used by consumer to check the remote changes.
-	// Default: 1m
-	Interval time.Duration
-
-	// AfterSync callbacks are triggered after each sync, receiving
-	// the sync state and error (if occurred).
-	AfterSync func(*ConsumerSync, error)
-}
-
-func (o *ConsumerOptions) norm(name string) {
-	o.ReaderOptions.norm(name)
-	if o.Interval <= 0 {
-		o.Interval = time.Minute
-	}
-}
-
-// ConsumerSync contains the state of the last sync.
-type ConsumerSync struct {
-	// Consumer exposes the current consumer state.
-	Consumer
-	// Updated indicates is the sync resulted in an update.
-	Updated bool
-	// PreviousData references the data before the update.
-	// It allows to apply finalizers to data structures created by ConsumeFunc.
-	// This is only set when an update happened.
-	PreviousData interface{}
-}
-
-// ConsumeFunc is a parsing callback which is run by the consumer every sync interval.
-type ConsumeFunc func(*Reader) (data interface{}, err error)
+// ConsumeFunc is a callback invoked by consumers.
+type ConsumeFunc func(context.Context, *Reader) error
 
 // Consumer manages data retrieval from a remote feed.
 // It queries the feed in regular intervals, continuously retrieving new updates.
 type Consumer interface {
-	// Data returns the data as returned by ConsumeFunc on last sync.
-	Data() interface{}
-	// LastSync returns time of last sync attempt.
-	LastSync() time.Time
-	// LastConsumed returns time of last feed consumption.
-	LastConsumed() time.Time
-	// LastModified returns time at which the remote feed was last modified.
-	LastModified() time.Time
-	// NumRead returns the number of values consumed during the last sync.
-	NumRead() int
+	// Consume initiates a sync attempt. It will consume the remote feed only if it has changed since
+	// last invocation.
+	Consume(context.Context, *ReaderOptions, ConsumeFunc) (*Status, error)
+
+	// Version indicates the most recently consumed version.
+	Version() int64
+
 	// Close stops the underlying sync process.
 	Close() error
 }
 
 // NewConsumer starts a new feed consumer.
-func NewConsumer(ctx context.Context, remoteURL string, opt *ConsumerOptions, cfn ConsumeFunc) (Consumer, error) {
+func NewConsumer(ctx context.Context, remoteURL string) (Consumer, error) {
 	remote, err := bfs.NewObject(ctx, remoteURL)
 	if err != nil {
 		return nil, err
 	}
 
-	csm, err := NewConsumerForRemote(ctx, remote, opt, cfn)
-	if err != nil {
-		_ = remote.Close()
-		return nil, err
-	}
+	csm := NewConsumerForRemote(remote)
 	csm.(*consumer).ownRemote = true
 	return csm, nil
 }
 
 // NewConsumerForRemote starts a new feed consumer with a remote.
-func NewConsumerForRemote(ctx context.Context, remote *bfs.Object, opt *ConsumerOptions, cfn ConsumeFunc) (Consumer, error) {
-	var o ConsumerOptions
-	if opt != nil {
-		o = *opt
-	}
-	o.norm(remote.Name())
-
-	ctx, stop := context.WithCancel(ctx)
-	c := &consumer{
-		remote: remote,
-		opt:    o,
-		ctx:    ctx,
-		stop:   stop,
-		cfn:    cfn,
-	}
+func NewConsumerForRemote(remote *bfs.Object) Consumer {
+	return &consumer{remote: remote}
+}
 
-	// run initial sync
-	if _, err := c.sync(true); err != nil {
-		_ = c.Close()
+// NewIncrementalConsumer starts a new incremental feed consumer.
+func NewIncrementalConsumer(ctx context.Context, bucketURL string) (Consumer, error) {
+	bucket, err := bfs.Connect(ctx, bucketURL)
+	if err != nil {
 		return nil, err
 	}
 
-	// start continuous loop
-	go c.loop()
+	csm := NewIncrementalConsumerForBucket(bucket)
+	csm.(*consumer).ownBucket = true
+	return csm, nil
+}
 
-	return c, nil
+// NewIncrementalConsumerForBucket starts a new incremental feed consumer with a bucket.
+func NewIncrementalConsumerForBucket(bucket bfs.Bucket) Consumer {
+	return &consumer{
+		remote:    bfs.NewObjectFromBucket(bucket, "manifest.json"),
+		ownRemote: true,
+		bucket:    bucket,
+	}
 }
 
 type consumer struct {
 	remote    *bfs.Object
 	ownRemote bool
 
-	opt  ConsumerOptions
-	ctx  context.Context
-	stop context.CancelFunc
-
-	cfn  ConsumeFunc
-	data atomic.Value
-
-	numRead, lastMod, lastSync, lastConsumed int64
-}
-
-// Data implements Consumer interface.
-func (c *consumer) Data() interface{} {
-	return c.data.Load()
-}
-
-// NumRead implements Consumer interface.
-func (c *consumer) NumRead() int {
-	return int(atomic.LoadInt64(&c.numRead))
-}
-
-// LastSync implements Consumer interface.
-func (c *consumer) LastSync() time.Time {
-	return timestamp(atomic.LoadInt64(&c.lastSync)).Time()
-}
+	bucket    bfs.Bucket
+	ownBucket bool
 
-// LastConsumed implements Consumer interface.
-func (c *consumer) LastConsumed() time.Time {
-	return timestamp(atomic.LoadInt64(&c.lastConsumed)).Time()
+	version atomic.Int64
 }
 
-// LastModified implements Consumer interface.
-func (c *consumer) LastModified() time.Time {
-	return timestamp(atomic.LoadInt64(&c.lastMod)).Time()
-}
-
-// Close implements Consumer interface.
-func (c *consumer) Close() error {
-	c.stop()
-	if c.ownRemote {
-		return c.remote.Close()
+// Consume implements Consumer interface.
+func (c *consumer) Consume(ctx context.Context, opt *ReaderOptions, fn ConsumeFunc) (*Status, error) {
+	localVersion := c.Version()
+	status := Status{
+		LocalVersion: localVersion,
 	}
-	return nil
-}
 
-func (c *consumer) sync(force bool) (*ConsumerSync, error) {
-	syncTime := timestampFromTime(time.Now()).Millis()
-	defer func() {
-		atomic.StoreInt64(&c.lastSync, syncTime)
-	}()
-
-	// retrieve original last modified time
-	lastMod, err := remoteLastModified(c.ctx, c.remote)
+	// retrieve remote mtime
+	remoteVersion, err := fetchRemoteVersion(ctx, c.remote)
 	if err != nil {
 		return nil, err
 	}
+	status.RemoteVersion = remoteVersion
 
-	// skip update if not forced or modified
-	if !force && lastMod > 0 && lastMod.Millis() == atomic.LoadInt64(&c.lastMod) {
-		return &ConsumerSync{Consumer: c}, nil
+	// skip sync unless modified
+	if skipSync(remoteVersion, localVersion) {
+		status.Skipped = true
+		return &status, nil
 	}
 
-	// open remote reader
-	reader, err := NewReader(c.ctx, c.remote, &c.opt.ReaderOptions)
-	if err != nil {
-		return nil, err
+	var reader *Reader
+	if c.isIncremental() {
+		if reader, err = c.newIncrementalReader(ctx, opt); err != nil {
+			return nil, err
+		}
+	} else {
+		if reader, err = NewReader(ctx, c.remote, opt); err != nil {
+			return nil, err
+		}
 	}
 	defer reader.Close()
 
 	// consume feed
-	data, err := c.cfn(reader)
-	if err != nil {
+	if err := fn(ctx, reader); err != nil {
 		return nil, err
 	}
 
-	// update stores
-	previous := c.data.Load()
-	c.data.Store(data)
-	atomic.StoreInt64(&c.numRead, int64(reader.NumRead()))
-	atomic.StoreInt64(&c.lastMod, lastMod.Millis())
-	atomic.StoreInt64(&c.lastConsumed, syncTime)
-	return &ConsumerSync{
-		Consumer:     c,
-		Updated:      true,
-		PreviousData: previous,
-	}, nil
+	status.NumItems = reader.NumRead()
+	c.version.Store(remoteVersion)
+	return &status, nil
 }
 
-func (c *consumer) loop() {
-	ticker := time.NewTicker(c.opt.Interval)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-c.ctx.Done():
-			return
-		case <-ticker.C:
-			state, err := c.sync(false)
-			if c.opt.AfterSync != nil {
-				c.opt.AfterSync(state, err)
-			}
+// Version implements Consumer interface.
+func (c *consumer) Version() int64 {
+	return c.version.Load()
+}
+
+// Close implements Consumer interface.
+func (c *consumer) Close() (err error) {
+	if c.ownRemote && c.remote != nil {
+		if e := c.remote.Close(); e != nil {
+			err = errors.Join(err, e)
+		}
+		c.remote = nil
+	}
+	if c.ownBucket && c.bucket != nil {
+		if e := c.bucket.Close(); e != nil {
+			err = errors.Join(err, e)
 		}
+		c.bucket = nil
+	}
+	return
+}
+
+func (c *consumer) isIncremental() bool {
+	return c.bucket != nil
+}
+
+func (c *consumer) newIncrementalReader(ctx context.Context, opt *ReaderOptions) (*Reader, error) {
+	manifest, err := loadManifest(ctx, c.remote)
+	if err != nil {
+		return nil, err
+	}
+
+	files := manifest.Files
+	remotes := make([]*bfs.Object, 0, len(files))
+	for _, file := range files {
+		remotes = append(remotes, bfs.NewObjectFromBucket(c.bucket, file))
 	}
+	r := MultiReader(ctx, remotes, opt)
+	r.ownRemotes = true
+	return r, nil
 }

data/consumer_test.go

@@ -2,83 +2,148 @@ package feedx_test
 
 import (
 	"context"
-	"io"
-	"time"
+	"reflect"
+	"testing"
 
 	"github.com/bsm/bfs"
 	"github.com/bsm/feedx"
 	"github.com/bsm/feedx/internal/testdata"
-	. "github.com/bsm/ginkgo"
-	. "github.com/bsm/gomega"
 )
 
-var _ = Describe("Consumer", func() {
-	var subject feedx.Consumer
-	var obj *bfs.Object
-	var ctx = context.Background()
-
-	consume := func(r *feedx.Reader) (interface{}, error) {
-		var msgs []*testdata.MockMessage
-		for {
-			var msg testdata.MockMessage
-			if err := r.Decode(&msg); err == io.EOF {
-				break
-			} else if err != nil {
-				return nil, err
-			}
-			msgs = append(msgs, &msg)
+func TestConsumer(t *testing.T) {
+	t.Run("consumes", func(t *testing.T) {
+		csm := fixConsumer(t, 101)
+		defer csm.Close()
+
+		if exp, got := int64(0), csm.Version(); exp != got {
+			t.Errorf("expected %v, got %v", exp, got)
 		}
-		return msgs, nil
-	}
 
-	BeforeEach(func() {
-		obj = bfs.NewInMemObject("path/to/file.jsonz")
-		Expect(writeMulti(obj, 2, mockTime)).To(Succeed())
+		// first attempt
+		msgs := testConsume(t, csm, &feedx.Status{
+			LocalVersion:  0,
+			RemoteVersion: 101,
+			Skipped:       false,
+			NumItems:      2,
+		})
+		if exp, got := int64(101), csm.Version(); exp != got {
+			t.Errorf("expected %v, got %v", exp, got)
+		}
+		if exp, got := 2, len(msgs); exp != got {
+			t.Errorf("expected %v, got %v", exp, got)
+		}
 
-		var err error
-		subject, err = feedx.NewConsumerForRemote(ctx, obj, nil, consume)
-		Expect(err).NotTo(HaveOccurred())
+		// second attempt
+		_ = testConsume(t, csm, &feedx.Status{
+			LocalVersion:  101,
+			RemoteVersion: 101,
+			Skipped:       true,
+			NumItems:      0,
+		})
 	})
 
-	AfterEach(func() {
-		Expect(subject.Close()).To(Succeed())
-	})
+	t.Run("always if no version", func(t *testing.T) {
+		csm := fixConsumer(t, 0)
+		defer csm.Close()
 
-	It("syncs/retrieves feeds from remote", func() {
-		Expect(subject.LastSync()).To(BeTemporally("~", time.Now(), time.Second))
-		Expect(subject.LastConsumed()).To(BeTemporally("==", subject.LastSync()))
-		Expect(subject.LastModified()).To(BeTemporally("==", mockTime.Truncate(time.Millisecond)))
-		Expect(subject.NumRead()).To(Equal(2))
-		Expect(subject.Data()).To(ConsistOf(seed(), seed()))
-		Expect(subject.Close()).To(Succeed())
+		testConsume(t, csm, &feedx.Status{NumItems: 2})
+		testConsume(t, csm, &feedx.Status{NumItems: 2})
 	})
 
-	It("consumes feeds only if necessary", func() {
-		prevSync := subject.LastSync()
-		time.Sleep(2 * time.Millisecond)
+	t.Run("incremental", func(t *testing.T) {
+		csm := fixIncrementalConsumer(t, 101)
+		defer csm.Close()
+
+		// first attempt
+		msgs := testConsume(t, csm, &feedx.Status{
+			LocalVersion:  0,
+			RemoteVersion: 101,
+			NumItems:      4,
+		})
+		if exp, got := int64(101), csm.Version(); exp != got {
+			t.Errorf("expected %v, got %v", exp, got)
+		}
+		if exp, got := 4, len(msgs); exp != got {
+			t.Errorf("expected %v, got %v", exp, got)
+		}
 
-		testable := subject.(interface{ TestSync() error })
-		Expect(testable.TestSync()).To(Succeed())
-		Expect(subject.LastSync()).To(BeTemporally(">", prevSync))
-		Expect(subject.LastConsumed()).To(BeTemporally("==", prevSync)) // skipped on last sync
-		Expect(subject.LastModified()).To(BeTemporally("==", mockTime.Truncate(time.Millisecond)))
-		Expect(subject.NumRead()).To(Equal(2))
+		// second attempt
+		_ = testConsume(t, csm, &feedx.Status{
+			LocalVersion:  101,
+			RemoteVersion: 101,
+			Skipped:       true,
+		})
 	})
 
-	It("always consumes if LastModified not set", func() {
-		noModTime := bfs.NewInMemObject("path/to/file.json")
-		Expect(writeMulti(noModTime, 2, time.Time{})).To(Succeed())
+}
+
+func fixConsumer(t *testing.T, version int64) feedx.Consumer {
+	t.Helper()
+
+	obj := bfs.NewInMemObject("path/to/file.json")
+	t.Cleanup(func() { _ = obj.Close() })
+
+	if err := writeN(obj, 2, version); err != nil {
+		t.Fatal("unexpected error", err)
+	}
+
+	csm := feedx.NewConsumerForRemote(obj)
+	t.Cleanup(func() { _ = csm.Close() })
+
+	return csm
+}
 
-		csmr, err := feedx.NewConsumerForRemote(ctx, noModTime, nil, consume)
-		Expect(err).NotTo(HaveOccurred())
+func fixIncrementalConsumer(t *testing.T, version int64) feedx.Consumer {
+	t.Helper()
 
-		prevSync := csmr.LastSync()
-		time.Sleep(2 * time.Millisecond)
+	bucket := bfs.NewInMem()
+	obj1 := bfs.NewObjectFromBucket(bucket, "data-0-0.json")
+	if err := writeN(obj1, 2, 0); err != nil {
+		t.Fatal("unexpected error", err)
+	}
+	defer obj1.Close()
+
+	obj2 := bfs.NewObjectFromBucket(bucket, "data-0-1.json")
+	if err := writeN(obj2, 2, 0); err != nil {
+		t.Fatal("unexpected error", err)
+	}
+	defer obj2.Close()
+
+	objm := bfs.NewObjectFromBucket(bucket, "manifest.json")
+	defer objm.Close()
+
+	manifest := &feedx.Manifest{
+		Version: version,
+		Files:   []string{obj1.Name(), obj2.Name()},
+	}
+	writer := feedx.NewWriter(t.Context(), objm, &feedx.WriterOptions{Version: version})
+	defer writer.Discard()
+
+	if err := writer.Encode(manifest); err != nil {
+		t.Fatal("unexpected error", err)
+	} else if err := writer.Commit(); err != nil {
+		t.Fatal("unexpected error", err)
+	}
+
+	csm := feedx.NewIncrementalConsumerForBucket(bucket)
+	t.Cleanup(func() { _ = csm.Close() })
+
+	return csm
+}
+
+func testConsume(t *testing.T, csm feedx.Consumer, exp *feedx.Status) (msgs []*testdata.MockMessage) {
+	t.Helper()
 
-		testable := csmr.(interface{ TestSync() error })
-		Expect(testable.TestSync()).To(Succeed())
-		Expect(csmr.LastSync()).To(BeTemporally(">", prevSync))
-		Expect(csmr.LastConsumed()).To(BeTemporally("==", csmr.LastSync())) // consumed on last sync
-		Expect(csmr.LastModified()).To(BeTemporally("==", time.Unix(0, 0)))
+	status, err := csm.Consume(t.Context(), nil, func(ctx context.Context, r *feedx.Reader) (err error) {
+		msgs, err = readMessages(r)
+		return err
 	})
-})
+	if err != nil {
+		t.Fatal("unexpected error", err)
+	}
+
+	if !reflect.DeepEqual(exp, status) {
+		t.Errorf("expected %#v, got %#v", exp, status)
+	}
+	return
+}

data/example_test.go

@@ -0,0 +1,140 @@
+package feedx_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/bsm/bfs"
+	"github.com/bsm/feedx"
+	"github.com/bsm/feedx/internal/testdata"
+)
+
+type message = testdata.MockMessage
+
+func Example() {
+	ctx := context.TODO()
+
+	// create an mock object
+	obj := bfs.NewInMemObject("todos.ndjson")
+	defer obj.Close()
+
+	pcr := feedx.NewProducerForRemote(obj)
+	defer pcr.Close()
+
+	// produce
+	status, err := pcr.Produce(ctx, 101, nil, func(w *feedx.Writer) error {
+		return errors.Join(
+			w.Encode(&message{Name: "Jane", Height: 175}),
+			w.Encode(&message{Name: "Joe", Height: 172}),
+		)
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Printf("PRODUCED skipped:%v version:%v->%v items:%v\n", status.Skipped, status.LocalVersion, status.RemoteVersion, status.NumItems)
+
+	// create a consumer
+	csm := feedx.NewConsumerForRemote(obj)
+	defer csm.Close()
+
+	// consume data
+	var msgs []*message
+	status, err = csm.Consume(context.TODO(), nil, func(ctx context.Context, r *feedx.Reader) error {
+		for {
+			var msg message
+			if err := r.Decode(&msg); err != nil {
+				if errors.Is(err, io.EOF) {
+					break
+				}
+				return err
+			}
+			msgs = append(msgs, &msg)
+		}
+
+		return nil
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Printf("CONSUMED skipped:%v version:%v->%v items:%v\n", status.Skipped, status.LocalVersion, status.RemoteVersion, status.NumItems)
+	fmt.Printf("DATA     [%q, %q]\n", msgs[0].Name, msgs[1].Name)
+
+	// Output:
+	// PRODUCED skipped:false version:101->0 items:2
+	// CONSUMED skipped:false version:0->101 items:2
+	// DATA     ["Jane", "Joe"]
+}
+
+func ExampleScheduler_Consume() {
+	ctx := context.TODO()
+
+	// create an mock object
+	obj := bfs.NewInMemObject("todos.ndjson")
+	defer obj.Close()
+
+	// create a consumer
+	csm := feedx.NewConsumerForRemote(obj)
+	defer csm.Close()
+
+	job := feedx.Every(time.Hour).
+		WithContext(ctx).
+		BeforeSync(func(_ int64) bool {
+			fmt.Println("1. Before sync")
+			return true
+		}).
+		AfterSync(func(_ *feedx.Status, err error) {
+			fmt.Printf("3. After sync - error:%v", err)
+		}).
+		Consume(csm, func(_ context.Context, _ *feedx.Reader) error {
+			fmt.Println("2. Consuming feed")
+			return nil
+		})
+	job.Stop()
+
+	// Output:
+	// 1. Before sync
+	// 2. Consuming feed
+	// 3. After sync - error:<nil>
+}
+
+func ExampleScheduler_Produce() {
+	ctx := context.TODO()
+
+	// create an mock object
+	obj := bfs.NewInMemObject("todos.ndjson")
+	defer obj.Close()
+
+	// create a producer
+	pcr := feedx.NewProducerForRemote(obj)
+	defer pcr.Close()
+
+	job := feedx.Every(time.Hour).
+		WithContext(ctx).
+		BeforeSync(func(_ int64) bool {
+			fmt.Println("2. Before sync")
+			return true
+		}).
+		AfterSync(func(_ *feedx.Status, err error) {
+			fmt.Printf("4. After sync - error:%v", err)
+		}).
+		WithVersionCheck(func(_ context.Context) (int64, error) {
+			fmt.Println("1. Retrieve latest version")
+			return 101, nil
+		}).
+		Produce(pcr, func(w *feedx.Writer) error {
+			fmt.Println("3. Producing feed")
+			return nil
+		})
+	job.Stop()
+
+	// Output:
+	// 1. Retrieve latest version
+	// 2. Before sync
+	// 3. Producing feed
+	// 4. After sync - error:<nil>
+}

data/feedx.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'feedx'
-  s.version       = '0.12.7'
+  s.version       = '0.14.0'
   s.authors       = ['Black Square Media Ltd']
   s.email         = ['info@blacksquaremedia.com']
   s.summary       = %(Exchange data between components via feeds)
@@ -9,17 +9,9 @@ Gem::Specification.new do |s|
   s.license       = 'Apache-2.0'
 
   s.files         = `git ls-files -z`.split("\x0").reject {|f| f.start_with?('spec/') }
-  s.test_files    = `git ls-files -z -- spec/*`.split("\x0")
   s.require_paths = ['lib']
-  s.required_ruby_version = '>= 2.7'
+  s.required_ruby_version = '>= 3.2'
 
   s.add_dependency 'bfs', '>= 0.8.0'
-
-  s.add_development_dependency 'bundler'
-  s.add_development_dependency 'pbio'
-  s.add_development_dependency 'rake'
-  s.add_development_dependency 'red-parquet', '>= 3.0', '< 4.0'
-  s.add_development_dependency 'rspec'
-  s.add_development_dependency 'rubocop-bsm'
   s.metadata['rubygems_mfa_required'] = 'true'
 end