feedx 0.3.2 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d1c12376b32188b646923935967905bb97ab8c86dde28236a0341ee878eb631
-  data.tar.gz: 3c76d4356a0f6aafcad0e8ea37a2159f8170f5d1cea4072753e3c9947afd7613
+  metadata.gz: 67b4f1f345ff01e33e63e66daed1810694330cfe9158b60886f29ae6f98d169c
+  data.tar.gz: 684f5f44a347d13cc4f71aa31741e1fe5c477205d870a0db96555364b166b235
 SHA512:
-  metadata.gz: d55b8ee3620301c3ffc5c95d2895d00f54ee562250c4aa8236ed93fd99e552cd67c126feab4a17b063415518e46b113ef918491b19e37043b5d36fa9e09c2808
-  data.tar.gz: b8eee064c4e12e462097087431cdd9b435720eead2d7524c9cc7d44cb4a7dde410756f443b71cd5b24261d3b06c95f1d3975ddca6b6ff69692e0e9a77a742446
+  metadata.gz: 0a8a9c98a96d79644a1e36d9035cc17f92e3b41c372424fd4a46180e9517da0090c8a123d30d8bfd959a9726ee92b9b71dbfa0a7992fd3171642b1bfd3fa364e
+  data.tar.gz: ba6dcde4c14857f3f024a558ba64050d7eca53cafb66599fef5e39e1a9b3ed6e036fb1f94e43217a77d7cba47cd307b21ea5145a25f51131feeace142976a6b8

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    feedx (0.3.2)
+    feedx (0.4.0)
       bfs (>= 0.3.4)
 
 GEM

data/README.md

@@ -5,15 +5,15 @@
 
 Feed-based data exchange between services.
 
-## Usage
+## Usage (Ruby)
 
 ```ruby
 require 'bfs/s3'
 require 'feedx'
 
-# Init a new pusher with an S3 destination
+# Init a new producer with an S3 destination
 relation = Post.includes(:author)
-pusher = Feedx::Pusher.new relation, 's3://my-bucket/feeds/users.json.gz'
+producer = Feedx::Producer.new relation, 's3://my-bucket/feeds/users.json.gz'
 
 # Push a new feed every hour
 loop do

data/consumer.go

@@ -2,91 +2,84 @@ package feedx
 
 import (
 	"context"
-	"fmt"
-	"strconv"
 	"sync/atomic"
 	"time"
 
 	"github.com/bsm/bfs"
 )
 
-// ConsumerOptions configure the Puller instance.
+// ConsumerOptions configure the consumer instance.
 type ConsumerOptions struct {
-	// The interval used by Puller to check the remote changes.
+	ReaderOptions
+
+	// The interval used by consumer to check the remote changes.
 	// Default: 1m
 	Interval time.Duration
 
-	// Format specifies the format
-	// Default: auto-detected from URL path.
-	Format Format
-
-	// Compression specifies the compression type.
-	// Default: auto-detected from URL path.
-	Compression Compression
-
 	// AfterSync callbacks are triggered after each sync, receiving
 	// the updated status and error (if occurred).
 	AfterSync func(updated bool, err error)
 }
 
 func (o *ConsumerOptions) norm(name string) error {
+	o.ReaderOptions.norm(name)
 	if o.Interval <= 0 {
 		o.Interval = time.Minute
 	}
-	if o.Format == nil {
-		o.Format = DetectFormat(name)
-
-		if o.Format == nil {
-			return fmt.Errorf("feedx: unable to detect format from %q", name)
-		}
-	}
-	if o.Compression == nil {
-		o.Compression = DetectCompression(name)
-	}
 	return nil
 }
 
-// ParseFunc is a data parse function.
-type ParseFunc func(FormatDecoder) (data interface{}, size int64, err error)
+// ConsumeFunc is a parsing callback which is run by the consumer every sync interval.
+type ConsumeFunc func(FormatDecoder) (data interface{}, err error)
 
 // Consumer manages data retrieval from a remote feed.
 // It queries the feed in regular intervals, continuously retrieving new updates.
 type Consumer interface {
-	// Data returns the data as returned by ParseFunc on last sync.
+	// Data returns the data as returned by ConsumeFunc on last sync.
 	Data() interface{}
-	// LastCheck returns time of last sync attempt.
-	LastCheck() time.Time
+	// LastSync returns time of last sync attempt.
+	LastSync() time.Time
 	// LastModified returns time at which the remote feed was last modified.
 	LastModified() time.Time
-	// Size returns the size as returned by ParseFunc on last sync.
-	Size() int64
+	// NumRead returns the number of values consumed during the last sync.
+	NumRead() int
 	// Close stops the underlying sync process.
 	Close() error
 }
 
 // NewConsumer starts a new feed consumer.
-func NewConsumer(ctx context.Context, srcURL string, opt *ConsumerOptions, parse ParseFunc) (Consumer, error) {
-	src, err := bfs.NewObject(ctx, srcURL)
+func NewConsumer(ctx context.Context, remoteURL string, opt *ConsumerOptions, cfn ConsumeFunc) (Consumer, error) {
+	remote, err := bfs.NewObject(ctx, remoteURL)
+	if err != nil {
+		return nil, err
+	}
+
+	csm, err := NewConsumerForRemote(ctx, remote, opt, cfn)
 	if err != nil {
+		_ = remote.Close()
 		return nil, err
 	}
+	csm.(*consumer).ownRemote = true
+	return csm, nil
+}
 
+// NewConsumerForRemote starts a new feed consumer with a remote.
+func NewConsumerForRemote(ctx context.Context, remote *bfs.Object, opt *ConsumerOptions, cfn ConsumeFunc) (Consumer, error) {
 	var o ConsumerOptions
 	if opt != nil {
 		o = *opt
 	}
-	if err := o.norm(src.Name()); err != nil {
-		_ = src.Close()
+	if err := o.norm(remote.Name()); err != nil {
 		return nil, err
 	}
 
 	ctx, stop := context.WithCancel(ctx)
 	f := &consumer{
-		src:   src,
-		opt:   o,
-		ctx:   ctx,
-		stop:  stop,
-		parse: parse,
+		remote: remote,
+		opt:    o,
+		ctx:    ctx,
+		stop:   stop,
+		cfn:    cfn,
 	}
 
 	// run initial sync
@@ -102,91 +95,81 @@ func NewConsumer(ctx context.Context, srcURL string, opt *ConsumerOptions, parse
 }
 
 type consumer struct {
-	src  *bfs.Object
+	remote    *bfs.Object
+	ownRemote bool
+
 	opt  ConsumerOptions
 	ctx  context.Context
 	stop context.CancelFunc
 
-	parse ParseFunc
+	cfn  ConsumeFunc
+	data atomic.Value
 
-	size, lastModMs int64
-	data, lastCheck atomic.Value
+	numRead, lastMod, lastSync int64
 }
 
-// Data implements Feed interface.
+// Data implements Consumer interface.
 func (f *consumer) Data() interface{} {
 	return f.data.Load()
 }
 
-// Size implements Feed interface.
-func (f *consumer) Size() int64 {
-	return atomic.LoadInt64(&f.size)
+// NumRead implements Consumer interface.
+func (f *consumer) NumRead() int {
+	return int(atomic.LoadInt64(&f.numRead))
 }
 
-// LastCheck implements Feed interface.
-func (f *consumer) LastCheck() time.Time {
-	return f.lastCheck.Load().(time.Time)
+// LastSync implements Consumer interface.
+func (f *consumer) LastSync() time.Time {
+	return timestamp(atomic.LoadInt64(&f.lastSync)).Time()
 }
 
-// LastModified implements Feed interface.
+// LastModified implements Consumer interface.
 func (f *consumer) LastModified() time.Time {
-	msec := atomic.LoadInt64(&f.lastModMs)
-	return time.Unix(msec/1000, msec%1000*1e6)
+	return timestamp(atomic.LoadInt64(&f.lastMod)).Time()
 }
 
-// Close implements Feed interface.
+// Close implements Consumer interface.
 func (f *consumer) Close() error {
 	f.stop()
-	return f.src.Close()
+	if f.ownRemote {
+		return f.remote.Close()
+	}
+	return nil
 }
 
 func (f *consumer) sync(force bool) (bool, error) {
-	f.lastCheck.Store(time.Now())
+	defer func() {
+		atomic.StoreInt64(&f.lastSync, timestampFromTime(time.Now()).Millis())
+	}()
 
-	info, err := f.src.Head(f.ctx)
+	// retrieve original last modified time
+	lastMod, err := remoteLastModified(f.ctx, f.remote)
 	if err != nil {
 		return false, err
 	}
 
-	// calculate last modified time
-	msec, _ := strconv.ParseInt(info.Metadata[lastModifiedMetaKey], 10, 64)
-
 	// skip update if not forced or modified
-	if msec == atomic.LoadInt64(&f.lastModMs) && !force {
+	if lastMod.Millis() == atomic.LoadInt64(&f.lastMod) && !force {
 		return false, nil
 	}
 
-	// open remote for reading
-	r, err := f.src.Open(f.ctx)
-	if err != nil {
-		return false, err
-	}
-	defer r.Close()
-
-	// wrap in compressed reader
-	c, err := f.opt.Compression.NewReader(r)
-	if err != nil {
-		return false, err
-	}
-	defer c.Close()
-
-	// open decoder
-	d, err := f.opt.Format.NewDecoder(c)
+	// open remote reader
+	reader, err := NewReader(f.ctx, f.remote, &f.opt.ReaderOptions)
 	if err != nil {
 		return false, err
 	}
-	defer f.Close()
+	defer reader.Close()
 
-	// parse feed
-	data, size, err := f.parse(d)
+	// consume feed
+	data, err := f.cfn(reader)
 	if err != nil {
 		return false, err
 	}
 
 	// update stores
 	f.data.Store(data)
-	atomic.StoreInt64(&f.size, size)
-	atomic.StoreInt64(&f.lastModMs, msec)
+	atomic.StoreInt64(&f.numRead, int64(reader.NumRead()))
+	atomic.StoreInt64(&f.lastMod, lastMod.Millis())
 	return true, nil
 }
 
@@ -200,7 +183,9 @@ func (f *consumer) loop() {
 			return
 		case <-ticker.C:
 			updated, err := f.sync(false)
-			f.opt.AfterSync(updated, err)
+			if f.opt.AfterSync != nil {
+				f.opt.AfterSync(updated, err)
+			}
 		}
 	}
 }

data/consumer_test.go

@@ -13,58 +13,41 @@ import (
 )
 
 var _ = Describe("Consumer", func() {
-	ctx := context.Background()
-	msg := &tbp.Message{
-		Name:         "Joe",
-		TrueScotsman: true,
-		Hilarity:     tbp.Message_BILL_BAILEY,
-	}
-	pfn := func(dec feedx.FormatDecoder) (interface{}, int64, error) {
-		var msgs []*tbp.Message
-		for {
-			msg := new(tbp.Message)
-			if err := dec.Decode(msg); err == io.EOF {
-				break
-			} else if err != nil {
-				return nil, 0, err
-			}
-			msgs = append(msgs, msg)
-		}
-		return msgs, int64(len(msgs)), nil
-	}
+	var subject feedx.Consumer
+	var obj *bfs.Object
+	var ctx = context.Background()
 
 	BeforeEach(func() {
-		memStore = bfs.NewInMem()
-		w, err := memStore.Create(ctx, "path/to/file.jsonz", &bfs.WriteOptions{
-			Metadata: map[string]string{"x-feedx-pusher-last-modified": "1544477788899"},
+		obj = bfs.NewInMemObject("path/to/file.jsonz")
+		Expect(writeMulti(obj, 2)).To(Succeed())
+
+		var err error
+		subject, err = feedx.NewConsumerForRemote(ctx, obj, nil, func(dec feedx.FormatDecoder) (interface{}, error) {
+			var msgs []tbp.Message
+			for {
+				var msg tbp.Message
+				if err := dec.Decode(&msg); err == io.EOF {
+					break
+				}
+				if err != nil {
+					return nil, err
+				}
+				msgs = append(msgs, msg)
+			}
+			return msgs, nil
 		})
 		Expect(err).NotTo(HaveOccurred())
-		defer w.Close()
-
-		c, err := feedx.GZipCompression.NewWriter(w)
-		Expect(err).NotTo(HaveOccurred())
-		defer c.Close()
-
-		f, err := feedx.JSONFormat.NewEncoder(c)
-		Expect(err).NotTo(HaveOccurred())
-		defer f.Close()
+	})
 
-		Expect(f.Encode(msg)).To(Succeed())
-		Expect(f.Encode(msg)).To(Succeed())
-		Expect(f.Close()).To(Succeed())
-		Expect(c.Close()).To(Succeed())
-		Expect(w.Close()).To(Succeed())
+	AfterEach(func() {
+		Expect(subject.Close()).To(Succeed())
 	})
 
 	It("should sync and retrieve feeds from remote", func() {
-		subject, err := feedx.NewConsumer(ctx, "mem:///path/to/file.jsonz", nil, pfn)
-		Expect(err).NotTo(HaveOccurred())
-		defer subject.Close()
-
-		Expect(subject.LastCheck()).To(BeTemporally("~", time.Now(), time.Second))
-		Expect(subject.LastModified()).To(BeTemporally("~", time.Unix(1544477788, 0), time.Second))
-		Expect(subject.Size()).To(Equal(int64(2)))
-		Expect(subject.Data()).To(Equal([]*tbp.Message{msg, msg}))
+		Expect(subject.LastSync()).To(BeTemporally("~", time.Now(), time.Second))
+		Expect(subject.LastModified()).To(BeTemporally("~", time.Unix(1515151515, 0), time.Second))
+		Expect(subject.NumRead()).To(Equal(2))
+		Expect(subject.Data()).To(Equal([]tbp.Message{fixture, fixture}))
 		Expect(subject.Close()).To(Succeed())
 	})
 })

data/feedx.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'feedx'
-  s.version       = '0.3.2'
+  s.version       = '0.4.0'
   s.authors       = ['Black Square Media Ltd']
   s.email         = ['info@blacksquaremedia.com']
   s.summary       = %(Exchange data between components via feeds)

data/feedx.go

@@ -1,3 +1,57 @@
 package feedx
 
-const lastModifiedMetaKey = "x-feedx-pusher-last-modified"
+import (
+	"context"
+	"errors"
+	"strconv"
+	"time"
+
+	"github.com/bsm/bfs"
+)
+
+// ErrNotModified is used to signal that something has not been modified.
+var ErrNotModified = errors.New("feedx: not modified")
+
+const (
+	metaLastModified       = "x-feedx-last-modified"
+	metaPusherLastModified = "x-feedx-pusher-last-modified"
+)
+
+// Timestamp with millisecond resolution
+type timestamp int64
+
+func timestampFromTime(t time.Time) timestamp {
+	if n := t.Unix()*1000 + int64(t.Nanosecond()/1e6); n > 0 {
+		return timestamp(n)
+	}
+	return 0
+}
+
+func remoteLastModified(ctx context.Context, obj *bfs.Object) (timestamp, error) {
+	info, err := obj.Head(ctx)
+	if err == bfs.ErrNotFound {
+		return 0, nil
+	} else if err != nil {
+		return 0, err
+	}
+
+	millis, _ := strconv.ParseInt(info.Metadata[metaLastModified], 10, 64)
+	if millis == 0 {
+		millis, _ = strconv.ParseInt(info.Metadata[metaPusherLastModified], 10, 64)
+	}
+	return timestamp(millis), nil
+}
+
+// Millis returns the number of milliseconds since epoch.
+func (t timestamp) Millis() int64 { return int64(t) }
+
+// Time returns the time at t.
+func (t timestamp) Time() time.Time {
+	n := t.Millis()
+	return time.Unix(n/1000, n%1000*1e6)
+}
+
+// String returns a string of milliseconds.
+func (t timestamp) String() string {
+	return strconv.FormatInt(int64(t), 10)
+}

data/feedx_test.go

@@ -4,8 +4,11 @@ import (
 	"context"
 	"net/url"
 	"testing"
+	"time"
 
 	"github.com/bsm/bfs"
+	"github.com/bsm/feedx"
+	tbp "github.com/golang/protobuf/proto/proto3_proto"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -21,6 +24,30 @@ func init() {
 	})
 }
 
+var fixture = tbp.Message{
+	Name:       "Joe",
+	Hilarity:   tbp.Message_BILL_BAILEY,
+	HeightInCm: 180,
+}
+
+func writeMulti(obj *bfs.Object, numEntries int) error {
+	w, err := feedx.NewWriter(context.Background(), obj, &feedx.WriterOptions{
+		LastMod: time.Unix(1515151515, 123456789),
+	})
+	if err != nil {
+		return err
+	}
+	defer w.Close()
+
+	for i := 0; i < numEntries; i++ {
+		fix := fixture
+		if err := w.Encode(&fix); err != nil {
+			return err
+		}
+	}
+	return w.Close()
+}
+
 func TestSuite(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "feedx")

data/format_test.go

@@ -11,12 +11,6 @@ import (
 )
 
 var _ = Describe("Format", func() {
-	msg := &tbp.Message{
-		Name:         "Joe",
-		TrueScotsman: true,
-		Hilarity:     tbp.Message_BILL_BAILEY,
-	}
-
 	runSharedTest := func(subject feedx.Format) {
 		buf := new(bytes.Buffer)
 
@@ -24,8 +18,9 @@ var _ = Describe("Format", func() {
 		Expect(err).NotTo(HaveOccurred())
 		defer enc.Close()
 
-		Expect(enc.Encode(msg)).To(Succeed())
-		Expect(enc.Encode(msg)).To(Succeed())
+		fix := fixture
+		Expect(enc.Encode(&fix)).To(Succeed())
+		Expect(enc.Encode(&fix)).To(Succeed())
 		Expect(enc.Close()).To(Succeed())
 
 		dec, err := subject.NewDecoder(buf)

data/lib/feedx.rb

@@ -1,6 +1,9 @@
 module Feedx
+  META_LAST_MODIFIED = 'x-feedx-last-modified'.freeze
+
   autoload :Compression, 'feedx/compression'
   autoload :Format, 'feedx/format'
+  autoload :Producer, 'feedx/producer'
   autoload :Pusher, 'feedx/pusher'
   autoload :TaskState, 'feedx/task_state'
 end

data/lib/feedx/producer.rb

@@ -0,0 +1,84 @@
+require 'uri'
+require 'bfs'
+require 'feedx'
+
+module Feedx
+  # Produces a relation as am encoded stream to a remote location.
+  class Producer
+    # See constructor.
+    def self.perform(url, opts={}, &block)
+      new(url, opts, &block).perform
+    end
+
+    # @param [String] url the destination URL.
+    # @param [Hash] opts options
+    # @option opts [Enumerable,ActiveRecord::Relation] :enum relation or enumerator to stream.
+    # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
+    # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
+    # @option opts [Time,Proc] :last_modified the last modified time, used to determine if a push is necessary.
+    # @yield A block factory to generate the relation or enumerator.
+    # @yieldreturn [Enumerable,ActiveRecord::Relation] the relation or enumerator to stream.
+    def initialize(url, opts={}, &block)
+      @enum = opts[:enum] || block
+      raise ArgumentError, "#{self.class.name}.new expects an :enum option or a block factory" unless @enum
+
+      @blob     = BFS::Blob.new(url)
+      @format   = detect_format(opts[:format])
+      @compress = detect_compress(opts[:compress])
+      @last_mod = opts[:last_modified]
+    end
+
+    def perform
+      enum = @enum.is_a?(Proc) ? @enum.call : @enum
+      last_mod = @last_mod.is_a?(Proc) ? @last_mod.call(enum) : @last_mod
+      current  = (last_mod.to_f * 1000).floor
+
+      begin
+        previous = @blob.info.metadata[META_LAST_MODIFIED].to_i
+        return -1 unless current > previous
+      rescue BFS::FileNotFound # rubocop:disable Lint/HandleExceptions
+      end if current.positive?
+
+      @blob.create metadata: { META_LAST_MODIFIED => current.to_s } do |io|
+        @compress.wrap(io) {|w| write_all(enum, w) }
+      end
+      @blob.info.size
+    end
+
+    private
+
+    def detect_format(val)
+      case val
+      when nil
+        Feedx::Format.detect(@blob.path)
+      when Class
+        parent = Feedx::Format::Abstract
+        raise ArgumentError, "Class #{val} must extend #{parent}" unless val < parent
+
+        val
+      else
+        Feedx::Format.resolve(val)
+      end
+    end
+
+    def detect_compress(val)
+      case val
+      when nil
+        Feedx::Compression.detect(@blob.path)
+      when Class
+        parent = Feedx::Compression::Abstract
+        raise ArgumentError, "Class #{val} must extend #{parent}" unless val < parent
+
+        val
+      else
+        Feedx::Compression.resolve(val)
+      end
+    end
+
+    def write_all(enum, io)
+      stream   = @format.new(io)
+      iterator = enum.respond_to?(:find_each) ? :find_each : :each
+      enum.send(iterator) {|rec| stream.write(rec) }
+    end
+  end
+end

data/lib/feedx/pusher.rb

@@ -1,85 +1,8 @@
-require 'uri'
-require 'bfs'
+require 'feedx'
 
 module Feedx
-  # Pushes a relation as a protobuf encoded stream to an S3 location.
-  class Pusher
-    META_LAST_MODIFIED = 'x-feedx-pusher-last-modified'.freeze
-
-    # See constructor.
-    def self.perform(url, opts={}, &block)
-      new(url, opts, &block).perform
-    end
-
-    # @param [String] url the destination URL.
-    # @param [Hash] opts options
-    # @option opts [Enumerable,ActiveRecord::Relation] :enum relation or enumerator to stream.
-    # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
-    # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
-    # @option opts [Time,Proc] :last_modified the last modified time, used to determine if a push is necessary.
-    # @yield A block factory to generate the relation or enumerator.
-    # @yieldreturn [Enumerable,ActiveRecord::Relation] the relation or enumerator to stream.
-    def initialize(url, opts={}, &block)
-      @enum = opts[:enum] || block
-      raise ArgumentError, "#{self.class.name}.new expects an :enum option or a block factory" unless @enum
-
-      @blob     = BFS::Blob.new(url)
-      @format   = detect_format(opts[:format])
-      @compress = detect_compress(opts[:compress])
-      @last_mod = opts[:last_modified]
-    end
-
-    def perform
-      enum = @enum.is_a?(Proc) ? @enum.call : @enum
-      last_mod = @last_mod.is_a?(Proc) ? @last_mod.call(enum) : @last_mod
-      current  = (last_mod.to_f * 1000).floor
-
-      begin
-        previous = @blob.info.metadata[META_LAST_MODIFIED].to_i
-        return -1 unless current > previous
-      rescue BFS::FileNotFound # rubocop:disable Lint/HandleExceptions
-      end if current.positive?
-
-      @blob.create metadata: { META_LAST_MODIFIED => current.to_s } do |io|
-        @compress.wrap(io) {|w| write_all(enum, w) }
-      end
-      @blob.info.size
-    end
-
-    private
-
-    def detect_format(val)
-      case val
-      when nil
-        Feedx::Format.detect(@blob.path)
-      when Class
-        parent = Feedx::Format::Abstract
-        raise ArgumentError, "Class #{val} must extend #{parent}" unless val < parent
-
-        val
-      else
-        Feedx::Format.resolve(val)
-      end
-    end
-
-    def detect_compress(val)
-      case val
-      when nil
-        Feedx::Compression.detect(@blob.path)
-      when Class
-        parent = Feedx::Compression::Abstract
-        raise ArgumentError, "Class #{val} must extend #{parent}" unless val < parent
-
-        val
-      else
-        Feedx::Compression.resolve(val)
-      end
-    end
-
-    def write_all(enum, io)
-      stream   = @format.new(io)
-      iterator = enum.respond_to?(:find_each) ? :find_each : :each
-      enum.send(iterator) {|rec| stream.write(rec) }
-    end
+  unless defined?(Gem::Deprecate) && Gem::Deprecate.skip
+    warn "WARNING: Feedx::Pusher is deprecated; use Feedx::Producer instead (called from #{caller(2..2).first})."
   end
+  Pusher = Producer
 end

data/reader.go

@@ -0,0 +1,130 @@
+package feedx
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/bsm/bfs"
+)
+
+// ReaderOptions configure the reader instance.
+type ReaderOptions struct {
+	// Format specifies the format
+	// Default: auto-detected from URL path.
+	Format Format
+
+	// Compression specifies the compression type.
+	// Default: auto-detected from URL path.
+	Compression Compression
+}
+
+func (o *ReaderOptions) norm(name string) error {
+	if o.Format == nil {
+		o.Format = DetectFormat(name)
+
+		if o.Format == nil {
+			return fmt.Errorf("feedx: unable to detect format from %q", name)
+		}
+	}
+	if o.Compression == nil {
+		o.Compression = DetectCompression(name)
+	}
+	return nil
+}
+
+// Reader reads data from a remote feed.
+type Reader struct {
+	remote *bfs.Object
+	opt    ReaderOptions
+	ctx    context.Context
+	num    int
+
+	br io.ReadCloser // bfs reader
+	cr io.ReadCloser // compression reader
+	fd FormatDecoder
+}
+
+// NewReader inits a new reader.
+func NewReader(ctx context.Context, remote *bfs.Object, opt *ReaderOptions) (*Reader, error) {
+	var o ReaderOptions
+	if opt != nil {
+		o = *opt
+	}
+	if err := o.norm(remote.Name()); err != nil {
+		return nil, err
+	}
+
+	return &Reader{
+		remote: remote,
+		opt:    o,
+		ctx:    ctx,
+	}, nil
+}
+
+// Decode decodes the next value from the feed.
+func (r *Reader) Decode(v interface{}) error {
+	if r.br == nil {
+		br, err := r.remote.Open(r.ctx)
+		if err != nil {
+			return err
+		}
+		r.br = br
+	}
+
+	if r.cr == nil {
+		cr, err := r.opt.Compression.NewReader(r.br)
+		if err != nil {
+			return err
+		}
+		r.cr = cr
+	}
+
+	if r.fd == nil {
+		fd, err := r.opt.Format.NewDecoder(r.cr)
+		if err != nil {
+			return err
+		}
+		r.fd = fd
+	}
+
+	if err := r.fd.Decode(v); err != nil {
+		return err
+	}
+
+	r.num++
+	return nil
+}
+
+// NumRead returns the number of read values.
+func (r *Reader) NumRead() int {
+	return r.num
+}
+
+// LastModified returns the last modified time of the remote feed.
+func (r *Reader) LastModified() (time.Time, error) {
+	lastMod, err := remoteLastModified(r.ctx, r.remote)
+	return lastMod.Time(), err
+}
+
+// Close closes the reader.
+func (r *Reader) Close() error {
+	var err error
+	if r.fd != nil {
+		if e := r.fd.Close(); e != nil {
+			err = e
+		}
+	}
+	if r.cr != nil {
+		if e := r.cr.Close(); e != nil {
+			err = e
+		}
+	}
+	if r.br != nil {
+		if e := r.br.Close(); e != nil {
+			err = e
+		}
+	}
+	return err
+}

data/reader_test.go

@@ -0,0 +1,48 @@
+package feedx_test
+
+import (
+	"context"
+	"io"
+
+	"github.com/bsm/feedx"
+
+	"github.com/bsm/bfs"
+	tbp "github.com/golang/protobuf/proto/proto3_proto"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Reader", func() {
+	var subject *feedx.Reader
+	var obj *bfs.Object
+	var ctx = context.Background()
+
+	BeforeEach(func() {
+		obj = bfs.NewInMemObject("path/to/file.json")
+		Expect(writeMulti(obj, 3)).To(Succeed())
+
+		var err error
+		subject, err = feedx.NewReader(ctx, obj, nil)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		Expect(subject.Close()).To(Succeed())
+	})
+
+	It("should read", func() {
+		var msgs []tbp.Message
+		for {
+			var msg tbp.Message
+			err := subject.Decode(&msg)
+			if err == io.EOF {
+				break
+			}
+			Expect(err).NotTo(HaveOccurred())
+			msgs = append(msgs, msg)
+		}
+
+		Expect(msgs).To(Equal([]tbp.Message{fixture, fixture, fixture}))
+		Expect(subject.NumRead()).To(Equal(3))
+	})
+})

data/spec/feedx/{pusher_spec.rb → producer_spec.rb}

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-RSpec.describe Feedx::Pusher do
+RSpec.describe Feedx::Producer do
   let :model do
     Class.new Struct.new(:title) do
       def to_pb
@@ -60,7 +60,7 @@ RSpec.describe Feedx::Pusher do
 
   it 'should support last-modified' do
     described_class.perform 'mock:///dir/file.json', last_modified: Time.at(1515151515), enum: enumerable
-    expect(bucket.info('dir/file.json').metadata).to eq('x-feedx-pusher-last-modified' => '1515151515000')
+    expect(bucket.info('dir/file.json').metadata).to eq('x-feedx-last-modified' => '1515151515000')
   end
 
   it 'should perform conditionally' do

data/writer.go

@@ -0,0 +1,135 @@
+package feedx
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/bsm/bfs"
+)
+
+// WriterOptions configure the producer instance.
+type WriterOptions struct {
+	// Format specifies the format
+	// Default: auto-detected from URL path.
+	Format Format
+
+	// Compression specifies the compression type.
+	// Default: auto-detected from URL path.
+	Compression Compression
+
+	// Provides an optional last modified timestamp which is stored with the remote metadata.
+	// Default: time.Now().
+	LastMod time.Time
+}
+
+func (o *WriterOptions) norm(name string) error {
+	if o.Format == nil {
+		o.Format = DetectFormat(name)
+
+		if o.Format == nil {
+			return fmt.Errorf("feedx: unable to detect format from %q", name)
+		}
+	}
+
+	if o.Compression == nil {
+		o.Compression = DetectCompression(name)
+	}
+
+	if o.LastMod.IsZero() {
+		o.LastMod = time.Now()
+	}
+
+	return nil
+}
+
+// Writer encodes feeds to remote locations.
+type Writer struct {
+	ctx    context.Context
+	remote *bfs.Object
+	opt    WriterOptions
+	num    int
+
+	bw io.WriteCloser // bfs writer
+	cw io.WriteCloser // compression writer
+	fe FormatEncoder
+}
+
+// NewWriter inits a new feed writer.
+func NewWriter(ctx context.Context, remote *bfs.Object, opt *WriterOptions) (*Writer, error) {
+	var o WriterOptions
+	if opt != nil {
+		o = *opt
+	}
+	o.norm(remote.Name())
+
+	return &Writer{
+		ctx:    ctx,
+		remote: remote,
+		opt:    o,
+	}, nil
+}
+
+// Encode appends a value to the feed.
+func (w *Writer) Encode(v interface{}) error {
+	if w.bw == nil {
+		ts := timestampFromTime(w.opt.LastMod)
+		bw, err := w.remote.Create(w.ctx, &bfs.WriteOptions{
+			Metadata: map[string]string{metaLastModified: ts.String()},
+		})
+		if err != nil {
+			return err
+		}
+		w.bw = bw
+	}
+
+	if w.cw == nil {
+		cw, err := w.opt.Compression.NewWriter(w.bw)
+		if err != nil {
+			return err
+		}
+		w.cw = cw
+	}
+
+	if w.fe == nil {
+		fe, err := w.opt.Format.NewEncoder(w.cw)
+		if err != nil {
+			return err
+		}
+		w.fe = fe
+	}
+
+	if err := w.fe.Encode(v); err != nil {
+		return err
+	}
+
+	w.num++
+	return nil
+}
+
+// NumWritten returns the number of written values.
+func (w *Writer) NumWritten() int {
+	return w.num
+}
+
+// Close closes the writer.
+func (w *Writer) Close() error {
+	var err error
+	if w.fe != nil {
+		if e := w.fe.Close(); e != nil {
+			err = e
+		}
+	}
+	if w.cw != nil {
+		if e := w.cw.Close(); e != nil {
+			err = e
+		}
+	}
+	if w.bw != nil {
+		if e := w.bw.Close(); e != nil {
+			err = e
+		}
+	}
+	return err
+}

data/writer_test.go

@@ -0,0 +1,34 @@
+package feedx_test
+
+import (
+	"context"
+
+	"github.com/bsm/bfs"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Writer", func() {
+	var plain, compressed *bfs.Object
+	var ctx = context.Background()
+
+	BeforeEach(func() {
+		plain = bfs.NewInMemObject("path/to/file.json")
+		compressed = bfs.NewInMemObject("path/to/file.jsonz")
+	})
+
+	It("should encode", func() {
+		Expect(writeMulti(plain, 10)).To(Succeed())
+		Expect(writeMulti(compressed, 10)).To(Succeed())
+
+		info, err := plain.Head(ctx)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(info.Size).To(BeNumerically("~", 470, 10))
+		Expect(info.Metadata).To(Equal(map[string]string{"x-feedx-last-modified": "1515151515123"}))
+
+		info, err = compressed.Head(ctx)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(info.Size).To(BeNumerically("~", 76, 10))
+		Expect(info.Metadata).To(Equal(map[string]string{"x-feedx-last-modified": "1515151515123"}))
+	})
+})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: feedx
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
 platform: ruby
 authors:
 - Black Square Media Ltd
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-14 00:00:00.000000000 Z
+date: 2018-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bfs
@@ -131,15 +131,20 @@ files:
 - lib/feedx/format/abstract.rb
 - lib/feedx/format/json.rb
 - lib/feedx/format/protobuf.rb
+- lib/feedx/producer.rb
 - lib/feedx/pusher.rb
+- reader.go
+- reader_test.go
 - spec/feedx/compression/gzip_spec.rb
 - spec/feedx/compression/none_spec.rb
 - spec/feedx/compression_spec.rb
 - spec/feedx/format/json_spec.rb
 - spec/feedx/format/protobuf_spec.rb
 - spec/feedx/format_spec.rb
-- spec/feedx/pusher_spec.rb
+- spec/feedx/producer_spec.rb
 - spec/spec_helper.rb
+- writer.go
+- writer_test.go
 homepage: https://github.com/bsm/feedx
 licenses:
 - Apache-2.0
@@ -171,5 +176,5 @@ test_files:
 - spec/feedx/format/json_spec.rb
 - spec/feedx/format/protobuf_spec.rb
 - spec/feedx/format_spec.rb
-- spec/feedx/pusher_spec.rb
+- spec/feedx/producer_spec.rb
 - spec/spec_helper.rb