feedx 0.12.5 → 0.12.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c1c91575fbae6b3b5df7c840d8355ec2d27916212a57d19192fe63288b39ed9e
-  data.tar.gz: e21dffcc3da9ebadd5f936380a00427339566bde0d058e66498528f9418942d6
+  metadata.gz: ab1b8d12efde331dcd1ca96c686c29e1ef16ab895f049121078b9f0503d23276
+  data.tar.gz: 3d51319d3ea39cbceeb480252e06c37238fce0cda68aa6b65cf1f59c0f0af38b
 SHA512:
-  metadata.gz: d8f0e86db26b77894b062a9919c6f902ddb6123f218fdb9233ab8126b0905c023ba8a32d69cc90b3f035345e637cb6719bd43bee984f3f8c62460772f941405f
-  data.tar.gz: 7451b7e3305ce7ec05a8f7c7d4a8cd9a8b65b1eef417f419903f139d01db94cc75045e0fac105eac4dc78699e2012c7c1450b16ad213615fe563799cb7ab1fab
+  metadata.gz: d08c34b59158c69d500b7cffe8019550dafa87b760e4b0cb3b25afc367801a3f3473486987dcb62e5c1bc62c05d3653bf26692fcbdcbb552d2af1073f4eced50
+  data.tar.gz: cfa3f24458cead0c12c1f03b559ded816095e1e9cf29a1918c142cd255f782fa3c51d7c45ca5833ab25b5a745e0e68f4b1b1062c7760c56ff1b57da122767135

data/Gemfile.lock

@@ -1,22 +1,15 @@
 PATH
   remote: .
   specs:
-    feedx (0.12.5)
+    feedx (0.12.6)
       bfs (>= 0.8.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.3.1)
-      concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 1.6, < 2)
-      minitest (>= 5.1)
-      tzinfo (~> 2.0)
-      zeitwerk (~> 2.3)
     ast (2.4.2)
-    bfs (0.8.1)
+    bfs (0.8.4)
     bigdecimal (3.0.2)
-    concurrent-ruby (1.1.8)
     diff-lcs (1.4.4)
     extpp (0.0.9)
     gio2 (3.4.4)
@@ -26,18 +19,14 @@ GEM
       pkg-config (>= 1.3.5)
     gobject-introspection (3.4.4)
       glib2 (= 3.4.4)
-    google-protobuf (3.15.8)
-    i18n (1.8.10)
-      concurrent-ruby (~> 1.0)
-    minitest (5.14.4)
+    google-protobuf (3.17.3)
     native-package-installer (1.1.1)
     parallel (1.20.1)
     parser (3.0.1.1)
       ast (~> 2.4.1)
-    pbio (0.2.2)
+    pbio (0.3.0)
       google-protobuf
     pkg-config (1.4.6)
-    rack (2.2.3)
     rainbow (3.0.0)
     rake (13.0.3)
     red-arrow (3.0.0)
@@ -63,40 +52,32 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
     rspec-support (3.10.2)
-    rubocop (1.13.0)
+    rubocop (1.17.0)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 1.2.0, < 2.0)
+      rubocop-ast (>= 1.7.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.5.0)
+    rubocop-ast (1.7.0)
       parser (>= 3.0.1.1)
-    rubocop-bsm (0.5.9)
+    rubocop-bsm (0.6.0)
       rubocop (~> 1.0)
       rubocop-performance
-      rubocop-rails
       rubocop-rake
       rubocop-rspec
-    rubocop-performance (1.11.1)
+    rubocop-performance (1.11.3)
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
-    rubocop-rails (2.9.1)
-      activesupport (>= 4.2.0)
-      rack (>= 1.1)
-      rubocop (>= 0.90.0, < 2.0)
     rubocop-rake (0.5.1)
       rubocop
-    rubocop-rspec (2.3.0)
+    rubocop-rspec (2.4.0)
       rubocop (~> 1.0)
       rubocop-ast (>= 1.1.0)
     ruby-progressbar (1.11.0)
-    tzinfo (2.0.4)
-      concurrent-ruby (~> 1.0)
     unicode-display_width (2.0.0)
-    zeitwerk (2.4.2)
 
 PLATFORMS
   ruby

data/compression_test.go

@@ -32,7 +32,7 @@ var _ = Describe("Compression", func() {
 		Expect(r.Close()).To(Succeed())
 	}
 
-	It("should detect the format", func() {
+	It("detects the format", func() {
 		Expect(feedx.DetectCompression("/path/to/file.json")).To(Equal(feedx.NoCompression))
 		Expect(feedx.DetectCompression("/path/to/file.json.gz")).To(Equal(feedx.GZipCompression))
 		Expect(feedx.DetectCompression("/path/to/file.jsonz")).To(Equal(feedx.GZipCompression))
@@ -53,7 +53,7 @@ var _ = Describe("Compression", func() {
 		var subject = feedx.NoCompression
 		var _ feedx.Compression = subject
 
-		It("should write/read", func() {
+		It("writes/reads", func() {
 			runSharedTest(subject)
 		})
 	})
@@ -62,7 +62,7 @@ var _ = Describe("Compression", func() {
 		var subject = feedx.GZipCompression
 		var _ feedx.Compression = subject
 
-		It("should write/read", func() {
+		It("writes/reads", func() {
 			runSharedTest(subject)
 		})
 	})
@@ -71,7 +71,7 @@ var _ = Describe("Compression", func() {
 		var subject = feedx.FlateCompression
 		var _ feedx.Compression = subject
 
-		It("should write/read", func() {
+		It("writes/reads", func() {
 			runSharedTest(subject)
 		})
 	})

data/consumer.go

@@ -50,6 +50,8 @@ type Consumer interface {
 	Data() interface{}
 	// LastSync returns time of last sync attempt.
 	LastSync() time.Time
+	// LastConsumed returns time of last feed consumption.
+	LastConsumed() time.Time
 	// LastModified returns time at which the remote feed was last modified.
 	LastModified() time.Time
 	// NumRead returns the number of values consumed during the last sync.
@@ -114,7 +116,7 @@ type consumer struct {
 	cfn  ConsumeFunc
 	data atomic.Value
 
-	numRead, lastMod, lastSync int64
+	numRead, lastMod, lastSync, lastConsumed int64
 }
 
 // Data implements Consumer interface.
@@ -132,6 +134,11 @@ func (c *consumer) LastSync() time.Time {
 	return timestamp(atomic.LoadInt64(&c.lastSync)).Time()
 }
 
+// LastConsumed implements Consumer interface.
+func (c *consumer) LastConsumed() time.Time {
+	return timestamp(atomic.LoadInt64(&c.lastConsumed)).Time()
+}
+
 // LastModified implements Consumer interface.
 func (c *consumer) LastModified() time.Time {
 	return timestamp(atomic.LoadInt64(&c.lastMod)).Time()
@@ -147,8 +154,9 @@ func (c *consumer) Close() error {
 }
 
 func (c *consumer) sync(force bool) (*ConsumerSync, error) {
+	syncTime := timestampFromTime(time.Now()).Millis()
 	defer func() {
-		atomic.StoreInt64(&c.lastSync, timestampFromTime(time.Now()).Millis())
+		atomic.StoreInt64(&c.lastSync, syncTime)
 	}()
 
 	// retrieve original last modified time
@@ -158,7 +166,7 @@ func (c *consumer) sync(force bool) (*ConsumerSync, error) {
 	}
 
 	// skip update if not forced or modified
-	if lastMod.Millis() == atomic.LoadInt64(&c.lastMod) && !force {
+	if !force && lastMod > 0 && lastMod.Millis() == atomic.LoadInt64(&c.lastMod) {
 		return &ConsumerSync{Consumer: c}, nil
 	}
 
@@ -180,6 +188,7 @@ func (c *consumer) sync(force bool) (*ConsumerSync, error) {
 	c.data.Store(data)
 	atomic.StoreInt64(&c.numRead, int64(reader.NumRead()))
 	atomic.StoreInt64(&c.lastMod, lastMod.Millis())
+	atomic.StoreInt64(&c.lastConsumed, syncTime)
 	return &ConsumerSync{
 		Consumer:     c,
 		Updated:      true,

data/consumer_test.go

@@ -17,25 +17,26 @@ var _ = Describe("Consumer", func() {
 	var obj *bfs.Object
 	var ctx = context.Background()
 
+	consume := func(r *feedx.Reader) (interface{}, error) {
+		var msgs []*testdata.MockMessage
+		for {
+			var msg testdata.MockMessage
+			if err := r.Decode(&msg); err == io.EOF {
+				break
+			} else if err != nil {
+				return nil, err
+			}
+			msgs = append(msgs, &msg)
+		}
+		return msgs, nil
+	}
+
 	BeforeEach(func() {
 		obj = bfs.NewInMemObject("path/to/file.jsonz")
-		Expect(writeMulti(obj, 2)).To(Succeed())
+		Expect(writeMulti(obj, 2, mockTime)).To(Succeed())
 
 		var err error
-		subject, err = feedx.NewConsumerForRemote(ctx, obj, nil, func(r *feedx.Reader) (interface{}, error) {
-			var msgs []*testdata.MockMessage
-			for {
-				var msg testdata.MockMessage
-				if err := r.Decode(&msg); err == io.EOF {
-					break
-				}
-				if err != nil {
-					return nil, err
-				}
-				msgs = append(msgs, &msg)
-			}
-			return msgs, nil
-		})
+		subject, err = feedx.NewConsumerForRemote(ctx, obj, nil, consume)
 		Expect(err).NotTo(HaveOccurred())
 	})
 
@@ -43,11 +44,41 @@ var _ = Describe("Consumer", func() {
 		Expect(subject.Close()).To(Succeed())
 	})
 
-	It("should sync and retrieve feeds from remote", func() {
+	It("syncs/retrieves feeds from remote", func() {
 		Expect(subject.LastSync()).To(BeTemporally("~", time.Now(), time.Second))
-		Expect(subject.LastModified()).To(BeTemporally("~", time.Unix(1515151515, 0), time.Second))
+		Expect(subject.LastConsumed()).To(BeTemporally("==", subject.LastSync()))
+		Expect(subject.LastModified()).To(BeTemporally("==", mockTime.Truncate(time.Millisecond)))
 		Expect(subject.NumRead()).To(Equal(2))
 		Expect(subject.Data()).To(ConsistOf(seed(), seed()))
 		Expect(subject.Close()).To(Succeed())
 	})
+
+	It("consumes feeds only if necessary", func() {
+		prevSync := subject.LastSync()
+		time.Sleep(2 * time.Millisecond)
+
+		testable := subject.(interface{ TestSync() error })
+		Expect(testable.TestSync()).To(Succeed())
+		Expect(subject.LastSync()).To(BeTemporally(">", prevSync))
+		Expect(subject.LastConsumed()).To(BeTemporally("==", prevSync)) // skipped on last sync
+		Expect(subject.LastModified()).To(BeTemporally("==", mockTime.Truncate(time.Millisecond)))
+		Expect(subject.NumRead()).To(Equal(2))
+	})
+
+	It("always consumes if LastModified not set", func() {
+		noModTime := bfs.NewInMemObject("path/to/file.json")
+		Expect(writeMulti(noModTime, 2, time.Time{})).To(Succeed())
+
+		csmr, err := feedx.NewConsumerForRemote(ctx, noModTime, nil, consume)
+		Expect(err).NotTo(HaveOccurred())
+
+		prevSync := csmr.LastSync()
+		time.Sleep(2 * time.Millisecond)
+
+		testable := csmr.(interface{ TestSync() error })
+		Expect(testable.TestSync()).To(Succeed())
+		Expect(csmr.LastSync()).To(BeTemporally(">", prevSync))
+		Expect(csmr.LastConsumed()).To(BeTemporally("==", csmr.LastSync())) // consumed on last sync
+		Expect(csmr.LastModified()).To(BeTemporally("==", time.Unix(0, 0)))
+	})
 })

data/ext/parquet/decoder_test.go

@@ -34,7 +34,7 @@ var _ = Describe("Decoder", func() {
 		Expect(fixture.Close()).To(Succeed())
 	})
 
-	It("should decode", func() {
+	It("decodes", func() {
 		v1 := new(mockStruct)
 		Expect(subject.Decode(v1)).To(Succeed())
 		Expect(v1).To(Equal(&mockStruct{
@@ -77,7 +77,7 @@ var _ = Describe("Decoder", func() {
 		Expect(subject.Decode(v9)).To(MatchError(io.EOF))
 	})
 
-	It("should open from non-file readers", func() {
+	It("opens from non-file readers", func() {
 		bin, err := ioutil.ReadFile("testdata/alltypes_plain.parquet")
 		Expect(err).NotTo(HaveOccurred())
 

data/ext/parquet/go.mod

@@ -3,7 +3,7 @@ module github.com/bsm/feedx/ext/parquet
 go 1.15
 
 require (
-	github.com/bsm/feedx v0.0.0-00010101000000-000000000000
+	github.com/bsm/feedx v0.12.5
 	github.com/bsm/ginkgo v1.16.1
 	github.com/bsm/gomega v1.11.0
 	github.com/bsm/parquet-go v0.0.0-20200904113737-792434824f00

data/ext/parquet/go.sum

@@ -4,6 +4,8 @@ github.com/bmatcuk/doublestar/v3 v3.0.0 h1:TQtVPlDnAYwcrVNB2JiGuMc++H5qzWZd9PhkN
 github.com/bmatcuk/doublestar/v3 v3.0.0/go.mod h1:6PcTVMw80pCY1RVuoqu3V++99uQB3vsSYKPTd8AWA0k=
 github.com/bsm/bfs v0.11.3 h1:BTFCftgmuVZwwu6vyjhyKr/Pg1E+cZ5tLodj3wKxr94=
 github.com/bsm/bfs v0.11.3/go.mod h1:sUhBrbc9g0XThRRrT9hiinMhhKbkKIdhLkFljk4fuzM=
+github.com/bsm/feedx v0.12.5 h1:N751MFTRKRgoP5eG1S30EoUuHsI7aAxJ0aQZg+7xf/Y=
+github.com/bsm/feedx v0.12.5/go.mod h1:l5YNhFomuWy9du+8+hznXMH8Hug2qSMik7b5Vipcy58=
 github.com/bsm/ginkgo v1.16.0/go.mod h1:RabIZLzOCPghgHJKUqHZpqrQETA5AnF4aCSIYy5C1bk=
 github.com/bsm/ginkgo v1.16.1 h1:jp1v1dbmbGZDWmnGXDTN+XK3U1fTTNja9xYa7VBI0l0=
 github.com/bsm/ginkgo v1.16.1/go.mod h1:RabIZLzOCPghgHJKUqHZpqrQETA5AnF4aCSIYy5C1bk=

data/feedx.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'feedx'
-  s.version       = '0.12.5'
+  s.version       = '0.12.6'
   s.authors       = ['Black Square Media Ltd']
   s.email         = ['info@blacksquaremedia.com']
   s.summary       = %(Exchange data between components via feeds)

data/feedx_ext_test.go

@@ -0,0 +1,6 @@
+package feedx
+
+func (c *consumer) TestSync() error {
+	_, err := c.sync(false)
+	return err
+}

data/feedx_test.go

@@ -32,12 +32,12 @@ func seed() *testdata.MockMessage {
 	}
 }
 
+var mockTime = time.Unix(1515151515, 123456789)
+
 // ------------------------------------------------------------------------
 
-func writeMulti(obj *bfs.Object, numEntries int) error {
-	w := feedx.NewWriter(context.Background(), obj, &feedx.WriterOptions{
-		LastMod: time.Unix(1515151515, 123456789),
-	})
+func writeMulti(obj *bfs.Object, numEntries int, lastMod time.Time) error {
+	w := feedx.NewWriter(context.Background(), obj, &feedx.WriterOptions{LastMod: lastMod})
 	defer w.Discard()
 
 	for i := 0; i < numEntries; i++ {

data/format_test.go

@@ -40,7 +40,7 @@ var _ = Describe("Format", func() {
 		Expect(dec.Close()).To(Succeed())
 	}
 
-	It("should detect the format", func() {
+	It("detects the format", func() {
 		Expect(feedx.DetectFormat("/path/to/file.json")).To(Equal(feedx.JSONFormat))
 		Expect(feedx.DetectFormat("/path/to/file.json.gz")).To(Equal(feedx.JSONFormat))
 		Expect(feedx.DetectFormat("/path/to/file.json.flate")).To(Equal(feedx.JSONFormat))
@@ -60,7 +60,7 @@ var _ = Describe("Format", func() {
 		var subject = feedx.JSONFormat
 		var _ feedx.Format = subject
 
-		It("should encode/decode", func() {
+		It("encodes/decodes", func() {
 			runSharedTest(subject)
 		})
 	})
@@ -69,7 +69,7 @@ var _ = Describe("Format", func() {
 		var subject = feedx.ProtobufFormat
 		var _ feedx.Format = subject
 
-		It("should encode/decode", func() {
+		It("encodes/decodes", func() {
 			runSharedTest(subject)
 		})
 	})

data/producer_test.go

@@ -43,7 +43,7 @@ var _ = Describe("Producer", func() {
 		}
 	})
 
-	It("should produce", func() {
+	It("produces", func() {
 		setup(nil)
 
 		Expect(subject.LastPush()).To(BeTemporally("~", time.Now(), time.Second))
@@ -56,7 +56,7 @@ var _ = Describe("Producer", func() {
 		Expect(info.Size).To(BeNumerically("~", 75, 10))
 	})
 
-	It("should produce with custom last-mod check", func() {
+	It("produces with custom last-mod check", func() {
 		setup(&feedx.ProducerOptions{
 			Interval:     50 * time.Millisecond,
 			LastModCheck: func(_ context.Context) (time.Time, error) { return time.Unix(1515151515, 987654321), nil },

data/reader_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"io"
 	"io/ioutil"
+	"time"
 
 	"github.com/bsm/bfs"
 	"github.com/bsm/feedx"
@@ -19,7 +20,7 @@ var _ = Describe("Reader", func() {
 
 	BeforeEach(func() {
 		obj = bfs.NewInMemObject("path/to/file.json")
-		Expect(writeMulti(obj, 3)).To(Succeed())
+		Expect(writeMulti(obj, 3, time.Time{})).To(Succeed())
 
 		var err error
 		subject, err = feedx.NewReader(ctx, obj, nil)
@@ -30,14 +31,14 @@ var _ = Describe("Reader", func() {
 		Expect(subject.Close()).To(Succeed())
 	})
 
-	It("should read", func() {
+	It("reads", func() {
 		data, err := ioutil.ReadAll(subject)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(len(data)).To(BeNumerically("~", 110, 20))
 		Expect(subject.NumRead()).To(Equal(0))
 	})
 
-	It("should decode", func() {
+	It("decodes", func() {
 		var msgs []*testdata.MockMessage
 		for {
 			var msg testdata.MockMessage

data/writer.go

@@ -20,7 +20,7 @@ type WriterOptions struct {
 	Compression Compression
 
 	// Provides an optional last modified timestamp which is stored with the remote metadata.
-	// Default: time.Now().
+	// Default: time.Time{}.
 	LastMod time.Time
 }
 
@@ -31,9 +31,6 @@ func (o *WriterOptions) norm(name string) {
 	if o.Compression == nil {
 		o.Compression = DetectCompression(name)
 	}
-	if o.LastMod.IsZero() {
-		o.LastMod = time.Now()
-	}
 }
 
 // Writer encodes feeds to remote locations.

data/writer_test.go

@@ -20,7 +20,7 @@ var _ = Describe("Writer", func() {
 		compressed = bfs.NewInMemObject("path/to/file.jsonz")
 	})
 
-	It("should write plain", func() {
+	It("writes plain", func() {
 		w := feedx.NewWriter(context.Background(), plain, &feedx.WriterOptions{
 			LastMod: time.Unix(1515151515, 123456789),
 		})
@@ -35,7 +35,7 @@ var _ = Describe("Writer", func() {
 		Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
 	})
 
-	It("should write compressed", func() {
+	It("writes compressed", func() {
 		w := feedx.NewWriter(context.Background(), compressed, &feedx.WriterOptions{
 			LastMod: time.Unix(1515151515, 123456789),
 		})
@@ -50,14 +50,14 @@ var _ = Describe("Writer", func() {
 		Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
 	})
 
-	It("should encode", func() {
-		Expect(writeMulti(plain, 10)).To(Succeed())
-		Expect(writeMulti(compressed, 10)).To(Succeed())
+	It("encodes", func() {
+		Expect(writeMulti(plain, 10, time.Time{})).To(Succeed())
+		Expect(writeMulti(compressed, 10, mockTime)).To(Succeed())
 
 		info, err := plain.Head(ctx)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(info.Size).To(BeNumerically("~", 370, 10))
-		Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
+		Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "0"}))
 
 		info, err = compressed.Head(ctx)
 		Expect(err).NotTo(HaveOccurred())

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: feedx
 version: !ruby/object:Gem::Version
-  version: 0.12.5
+  version: 0.12.6
 platform: ruby
 authors:
 - Black Square Media Ltd
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-05-04 00:00:00.000000000 Z
+date: 2021-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bfs
@@ -146,6 +146,7 @@ files:
 - ext/parquet/types.go
 - feedx.gemspec
 - feedx.go
+- feedx_ext_test.go
 - feedx_test.go
 - format.go
 - format_test.go