feedx 0.9.0 → 0.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4bc9eeecc2bf0ab3dbe9f3ff113908dea8a8dd666ac5dd07d16a71f0558af1ea
-  data.tar.gz: f9c939c92f7582f7e5b812112f51ccf6f3d153755121d45b381e9374df209d31
+  metadata.gz: c6a031f3dd500b1d022708f6997ce6b99fc1ca764c58d0d1a2d5c49430797975
+  data.tar.gz: 3cb5fa9951cd4918bb9191c8c644cb422b9c9d9a00944b5f68afc247b80a5935
 SHA512:
-  metadata.gz: 7011a7c08a47c6a747b3faee003625bfa998bb634f60dfa05f530deabbcba334c269f47b22d95d284335ce1fa63cb96ac2c134f7faf690483962ffd0214922c7
-  data.tar.gz: 5c42b111a8d440ec945939c923dc8fcfbc011964787154b240ae73309e7f0c6fe784f249ecba7b6001740f201460adf660ad72528e817d2546f457481e99fae4
+  metadata.gz: b2e0b49aafcc5b0f8ddad15b7738f815d097057ed205709a19ce8f734f1aaa50aaf10add1d580412c4fb673decb706c6a73094f8fb6d624e5de15a95c21c74a0
+  data.tar.gz: 96130e3b806c74c719fe6d31cbb4c3dcaa1dcbbdbb80e7a417df9d54a478b1eec166f6b4abeb4f665bf4c37c1f26a6640baaef3a9d3fb4c11afe233c6538a3a3

data/.travis.yml

@@ -15,8 +15,3 @@ matrix:
         - 1.13.x
       env:
         - GO111MODULE=on
-    - language: go
-      go:
-        - 1.12.x
-      env:
-        - GO111MODULE=on

data/Gemfile.lock

@@ -1,45 +1,45 @@
 PATH
   remote: .
   specs:
-    feedx (0.9.0)
+    feedx (0.9.2)
       bfs (>= 0.5.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.0)
-    bfs (0.5.1)
+    bfs (0.6.0)
     diff-lcs (1.3)
-    google-protobuf (3.10.0.rc.1)
-    jaro_winkler (1.5.3)
-    parallel (1.18.0)
-    parser (2.6.5.0)
+    google-protobuf (3.11.2)
+    jaro_winkler (1.5.4)
+    parallel (1.19.1)
+    parser (2.7.0.2)
       ast (~> 2.4.0)
     pbio (0.2.0)
       google-protobuf
     rainbow (3.0.0)
-    rake (13.0.0)
+    rake (13.0.1)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
       rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.0)
-      rspec-support (~> 3.9.0)
+    rspec-core (3.9.1)
+      rspec-support (~> 3.9.1)
     rspec-expectations (3.9.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.0)
+    rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
-    rspec-support (3.9.0)
-    rubocop (0.75.0)
+    rspec-support (3.9.2)
+    rubocop (0.79.0)
       jaro_winkler (~> 1.5.1)
       parallel (~> 1.10)
-      parser (>= 2.6)
+      parser (>= 2.7.0.1)
       rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 1.7)
-    rubocop-performance (1.5.0)
+    rubocop-performance (1.5.2)
       rubocop (>= 0.71.0)
     ruby-progressbar (1.10.1)
     unicode-display_width (1.6.0)
@@ -58,4 +58,4 @@ DEPENDENCIES
   rubocop-performance
 
 BUNDLED WITH
-   2.0.2
+   2.1.2

data/feedx.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'feedx'
-  s.version       = '0.9.0'
+  s.version       = '0.9.2'
   s.authors       = ['Black Square Media Ltd']
   s.email         = ['info@blacksquaremedia.com']
   s.summary       = %(Exchange data between components via feeds)

data/go.mod

@@ -1,8 +1,9 @@
 module github.com/bsm/feedx
 
 require (
-	github.com/bsm/bfs v0.8.0
-	github.com/gogo/protobuf v1.3.0
+	github.com/bmatcuk/doublestar v1.2.2 // indirect
+	github.com/bsm/bfs v0.8.1
+	github.com/gogo/protobuf v1.3.1
 	github.com/golang/protobuf v1.3.2
 	github.com/onsi/ginkgo v1.10.2
 	github.com/onsi/gomega v1.7.0

data/go.sum

@@ -1,11 +1,13 @@
 github.com/bmatcuk/doublestar v1.1.5 h1:2bNwBOmhyFEFcoB3tGvTD5xanq+4kyOZlB8wFYbMjkk=
 github.com/bmatcuk/doublestar v1.1.5/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
-github.com/bsm/bfs v0.8.0 h1:/suMKytZ3WhVY62osdFm6VD+gJtxaohlUBuf76vGC78=
-github.com/bsm/bfs v0.8.0/go.mod h1:cVv0jyqUY/jbHoG/WYPuWvOaOhW/HZ4jl7/JMlypvAE=
+github.com/bmatcuk/doublestar v1.2.2 h1:oC24CykoSAB8zd7XgruHo33E0cHJf/WhQA/7BeXj+x0=
+github.com/bmatcuk/doublestar v1.2.2/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
+github.com/bsm/bfs v0.8.1 h1:PS+kH6Pu8IsKUrxOp8E/B6/ASLvGEz2kIeX0GM4DdPc=
+github.com/bsm/bfs v0.8.1/go.mod h1:cVv0jyqUY/jbHoG/WYPuWvOaOhW/HZ4jl7/JMlypvAE=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE=
-github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=

data/lib/feedx/consumer.rb

@@ -8,7 +8,7 @@ module Feedx
     include Enumerable
 
     # See constructor.
-    def self.each(url, klass, opts={}, &block)
+    def self.each(url, klass, opts = {}, &block)
       new(url, klass, opts).each(&block)
     end
 
@@ -19,7 +19,7 @@ module Feedx
     # @option opts [Hash] :format_options format decode options. Default: {}.
     # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
     # @option opts [Feedx::Cache::Value] :cache cache value to store remote last modified time and consume conditionally.
-    def initialize(url, klass, opts={})
+    def initialize(url, klass, opts = {})
       @klass    = klass
       @stream   = Feedx::Stream.new(url, opts)
       @fmt_opts = opts[:format_options] || {}

data/lib/feedx/producer.rb

@@ -6,7 +6,7 @@ module Feedx
   # Produces a relation as an encoded feed to a remote location.
   class Producer
     # See constructor.
-    def self.perform(url, opts={}, &block)
+    def self.perform(url, opts = {}, &block)
       new(url, opts, &block).perform
     end
 
@@ -19,7 +19,7 @@ module Feedx
     # @option opts [Time,Proc] :last_modified the last modified time, used to determine if a push is necessary.
     # @yield A block factory to generate the relation or enumerator.
     # @yieldreturn [Enumerable,ActiveRecord::Relation] the relation or enumerator to stream.
-    def initialize(url, opts={}, &block)
+    def initialize(url, opts = {}, &block)
       @enum = opts[:enum] || block
       raise ArgumentError, "#{self.class.name}.new expects an :enum option or a block factory" unless @enum
 
@@ -37,7 +37,8 @@ module Feedx
         metadata   = @stream.blob.info.metadata
         remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
         return -1 unless local_rev > remote_rev
-      rescue BFS::FileNotFound # rubocop:disable Lint/HandleExceptions
+      rescue BFS::FileNotFound
+        nil
       end if local_rev.positive?
 
       @stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|

data/lib/feedx/stream.rb

@@ -10,7 +10,7 @@ module Feedx
     # @param [Hash] opts options
     # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
     # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
-    def initialize(url, opts={})
+    def initialize(url, opts = {})
       @blob     = BFS::Blob.new(url)
       @format   = detect_format(opts[:format])
       @compress = detect_compress(opts[:compress])
@@ -20,7 +20,7 @@ module Feedx
     # @param [Hash] opts BFS::Blob#open options
     # @yield A block over a formatted stream.
     # @yieldparam [Feedx::Format::Abstract] formatted input stream.
-    def open(opts={})
+    def open(opts = {})
       @blob.open(opts) do |io|
         @compress.reader(io) do |cio|
           fmt = @format.new(cio)
@@ -33,7 +33,7 @@ module Feedx
     # @param [Hash] opts BFS::Blob#create options
     # @yield A block over a formatted stream.
     # @yieldparam [Feedx::Format::Abstract] formatted output stream.
-    def create(opts={})
+    def create(opts = {})
       @blob.create(opts) do |io|
         @compress.writer(io) do |cio|
           fmt = @format.new(cio)

data/producer.go

@@ -140,14 +140,10 @@ func (p *Producer) push() (*ProducerPush, error) {
 		wopt.LastMod = modTime
 	}
 
-	// retrieve original last modified time
-	lastMod, err := remoteLastModified(p.ctx, p.remote)
-	if err != nil {
+	// retrieve original last modified time, skip if not modified
+	if rts, err := remoteLastModified(p.ctx, p.remote); err != nil {
 		return nil, err
-	}
-
-	// skip push if not modified
-	if lastMod.Time().Equal(wopt.LastMod) {
+	} else if rts == timestampFromTime(wopt.LastMod) {
 		return &ProducerPush{Producer: p}, nil
 	}
 

data/producer_test.go

@@ -2,6 +2,7 @@ package feedx_test
 
 import (
 	"context"
+	"sync/atomic"
 	"time"
 
 	"github.com/bsm/bfs"
@@ -13,11 +14,14 @@ import (
 var _ = Describe("Producer", func() {
 	var subject *feedx.Producer
 	var obj *bfs.Object
+	var numRuns uint32
 	var ctx = context.Background()
 
 	setup := func(o *feedx.ProducerOptions) {
 		var err error
 		subject, err = feedx.NewProducerForRemote(ctx, obj, o, func(w *feedx.Writer) error {
+			atomic.AddUint32(&numRuns, 1)
+
 			for i := 0; i < 10; i++ {
 				fix := fixture
 				if err := w.Encode(&fix); err != nil {
@@ -30,6 +34,7 @@ var _ = Describe("Producer", func() {
 	}
 
 	BeforeEach(func() {
+		atomic.StoreUint32(&numRuns, 0)
 		obj = bfs.NewInMemObject("path/to/file.jsonz")
 	})
 
@@ -54,16 +59,22 @@ var _ = Describe("Producer", func() {
 
 	It("should produce with custom last-mod check", func() {
 		setup(&feedx.ProducerOptions{
-			LastModCheck: func(_ context.Context) (time.Time, error) { return time.Unix(1515151515, 0), nil },
+			Interval:     50 * time.Millisecond,
+			LastModCheck: func(_ context.Context) (time.Time, error) { return time.Unix(1515151515, 987654321), nil },
 		})
 
-		Expect(subject.LastPush()).To(BeTemporally("~", time.Now(), time.Second))
-		Expect(subject.LastModified()).To(Equal(time.Unix(1515151515, 0)))
+		firstPush := subject.LastPush()
+		Expect(firstPush).To(BeTemporally("~", time.Now(), time.Second))
+		Expect(subject.LastModified()).To(Equal(time.Unix(1515151515, 987000000)))
 		Expect(subject.NumWritten()).To(Equal(10))
+		Expect(atomic.LoadUint32(&numRuns)).To(Equal(uint32(1)))
 
 		info, err := obj.Head(ctx)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(info.Size).To(BeNumerically("~", 75, 10))
-		Expect(info.Metadata).To(HaveKeyWithValue("X-Feedx-Last-Modified", "1515151515000"))
+		Expect(info.Metadata).To(HaveKeyWithValue("X-Feedx-Last-Modified", "1515151515987"))
+
+		Eventually(func() bool { return subject.LastPush().After(firstPush) }).Should(BeTrue())
+		Expect(atomic.LoadUint32(&numRuns)).To(Equal(uint32(1)))
 	})
 })

data/spec/feedx/consumer_spec.rb

@@ -36,7 +36,7 @@ RSpec.describe Feedx::Consumer do
 
   private
 
-  def mock_produce!(opts={})
+  def mock_produce!(opts = {})
     url = 'mock:///dir/file.json'
     opts[:enum] ||= %w[x y z].map {|t| Feedx::TestCase::Model.new(t) } * 100
     Feedx::Producer.perform url, opts

data/writer.go

@@ -3,6 +3,7 @@ package feedx
 import (
 	"bufio"
 	"context"
+	"errors"
 	"io"
 	"time"
 
@@ -114,7 +115,10 @@ func (w *Writer) NumWritten() int {
 // Discard closes the writer and discards the contents.
 func (w *Writer) Discard() error {
 	w.cancel()
-	return w.Commit()
+	if err := w.Commit(); err != nil && !errors.Is(err, context.Canceled) {
+		return err
+	}
+	return nil
 }
 
 // Commit closes the writer and persists the contents.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: feedx
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.9.2
 platform: ruby
 authors:
 - Black Square Media Ltd
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-10 00:00:00.000000000 Z
+date: 2020-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bfs
@@ -191,7 +191,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.6
 signing_key: 
 specification_version: 4
 summary: Exchange data between components via feeds