feedx 0.8.0 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/.travis.yml +4 -12
  4. data/Gemfile.lock +23 -20
  5. data/consumer_test.go +3 -4
  6. data/feedx.gemspec +2 -2
  7. data/feedx_test.go +27 -11
  8. data/format_test.go +3 -4
  9. data/go.mod +4 -3
  10. data/go.sum +26 -6
  11. data/lib/feedx/compression/gzip.rb +19 -5
  12. data/lib/feedx/consumer.rb +4 -4
  13. data/lib/feedx/producer.rb +6 -5
  14. data/lib/feedx/stream.rb +5 -5
  15. data/producer.go +4 -11
  16. data/producer_test.go +15 -4
  17. data/reader_test.go +4 -5
  18. data/spec/feedx/compression/gzip_spec.rb +2 -0
  19. data/spec/feedx/consumer_spec.rb +6 -3
  20. data/writer.go +20 -15
  21. data/writer_test.go +3 -5
  22. metadata +3 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 3d61930af3b031d191151820e5fefc106e3c64d8ece5d21513d8c91e4db195fb
4
- data.tar.gz: 005bf01fefeb8b33299063f9c4e132ee9f1237f19a8bbf35c907e78863474121
3
+ metadata.gz: 73577178d9531fd5397bca23cb9e85cb0d6ab8ea9f97e32b0ce5b97dbebe43d4
4
+ data.tar.gz: e1439be97329b54309831cd7c19c19982058e71879dbc612058f0bb5fce1a578
5
5
  SHA512:
6
- metadata.gz: a4027d850b2122075bbb4d3ddc776fcd1be28f2f5db3d22c57a348d76fd821630554fe3327ac486b47b9b83e72c590476a68ab12eebc373d0fac231bb7ce449f
7
- data.tar.gz: 857a3fc8cf13142a02634363e60478abd9171ef84c0d9a61f163c5debbea0a7a8378864d0e5ce27b4bdbe9a89d4b57df930c72a1da79602929a914a7ade38032
6
+ metadata.gz: 673e28b0f7d0e01543796f7aa5ea869a80e119c2fbbcbb21eb9d6918ea4e051ecc20e6f1634e79fcc56acf1991ae68b26f7f69e1ec43c47a8b96d8a359d20834
7
+ data.tar.gz: f2b9d69ef7dbc915c4a08c798e0da5f65a044907b27c3e742a2c4a7c1a1b5c514e648fc1924a335b99ca65d56454718b0a9041827306f6f19ce0a7c2d6507a97
data/.gitignore CHANGED
@@ -1,2 +1,3 @@
1
1
  .rubocop-*
2
2
  pkg/
3
+ *~
data/.travis.yml CHANGED
@@ -2,21 +2,13 @@ matrix:
2
2
  include:
3
3
  - language: ruby
4
4
  rvm:
5
- - 2.6
6
- before_install:
7
- - gem install bundler
5
+ - 2.7
8
6
  - language: ruby
9
7
  rvm:
10
- - 2.5
11
- before_install:
12
- - gem install bundler
8
+ - 2.6
13
9
  - language: go
14
10
  go:
15
- - 1.13.x
16
- env:
17
- - GO111MODULE=on
11
+ - 1.14.x
18
12
  - language: go
19
13
  go:
20
- - 1.12.x
21
- env:
22
- - GO111MODULE=on
14
+ - 1.13.x
data/Gemfile.lock CHANGED
@@ -1,48 +1,51 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- feedx (0.8.0)
4
+ feedx (0.10.2)
5
5
  bfs (>= 0.5.0)
6
6
 
7
7
  GEM
8
8
  remote: https://rubygems.org/
9
9
  specs:
10
10
  ast (2.4.0)
11
- bfs (0.5.0)
11
+ bfs (0.7.0)
12
12
  diff-lcs (1.3)
13
- google-protobuf (3.10.0-x86_64-linux)
14
- jaro_winkler (1.5.3)
15
- parallel (1.18.0)
16
- parser (2.6.5.0)
13
+ google-protobuf (3.12.2)
14
+ parallel (1.19.1)
15
+ parser (2.7.1.3)
17
16
  ast (~> 2.4.0)
18
- pbio (0.1.0)
17
+ pbio (0.2.1)
19
18
  google-protobuf
20
19
  rainbow (3.0.0)
21
- rake (13.0.0)
20
+ rake (13.0.1)
21
+ rexml (3.2.4)
22
22
  rspec (3.9.0)
23
23
  rspec-core (~> 3.9.0)
24
24
  rspec-expectations (~> 3.9.0)
25
25
  rspec-mocks (~> 3.9.0)
26
- rspec-core (3.9.0)
27
- rspec-support (~> 3.9.0)
28
- rspec-expectations (3.9.0)
26
+ rspec-core (3.9.2)
27
+ rspec-support (~> 3.9.3)
28
+ rspec-expectations (3.9.2)
29
29
  diff-lcs (>= 1.2.0, < 2.0)
30
30
  rspec-support (~> 3.9.0)
31
- rspec-mocks (3.9.0)
31
+ rspec-mocks (3.9.1)
32
32
  diff-lcs (>= 1.2.0, < 2.0)
33
33
  rspec-support (~> 3.9.0)
34
- rspec-support (3.9.0)
35
- rubocop (0.75.0)
36
- jaro_winkler (~> 1.5.1)
34
+ rspec-support (3.9.3)
35
+ rubocop (0.84.0)
37
36
  parallel (~> 1.10)
38
- parser (>= 2.6)
37
+ parser (>= 2.7.0.1)
39
38
  rainbow (>= 2.2.2, < 4.0)
39
+ rexml
40
+ rubocop-ast (>= 0.0.3)
40
41
  ruby-progressbar (~> 1.7)
41
- unicode-display_width (>= 1.4.0, < 1.7)
42
- rubocop-performance (1.5.0)
42
+ unicode-display_width (>= 1.4.0, < 2.0)
43
+ rubocop-ast (0.0.3)
44
+ parser (>= 2.7.0.1)
45
+ rubocop-performance (1.6.0)
43
46
  rubocop (>= 0.71.0)
44
47
  ruby-progressbar (1.10.1)
45
- unicode-display_width (1.6.0)
48
+ unicode-display_width (1.7.0)
46
49
 
47
50
  PLATFORMS
48
51
  ruby
@@ -58,4 +61,4 @@ DEPENDENCIES
58
61
  rubocop-performance
59
62
 
60
63
  BUNDLED WITH
61
- 2.0.2
64
+ 2.1.2
data/consumer_test.go CHANGED
@@ -7,7 +7,6 @@ import (
7
7
 
8
8
  "github.com/bsm/bfs"
9
9
  "github.com/bsm/feedx"
10
- tbp "github.com/golang/protobuf/proto/proto3_proto"
11
10
  . "github.com/onsi/ginkgo"
12
11
  . "github.com/onsi/gomega"
13
12
  )
@@ -23,9 +22,9 @@ var _ = Describe("Consumer", func() {
23
22
 
24
23
  var err error
25
24
  subject, err = feedx.NewConsumerForRemote(ctx, obj, nil, func(r *feedx.Reader) (interface{}, error) {
26
- var msgs []tbp.Message
25
+ var msgs []MockMessage
27
26
  for {
28
- var msg tbp.Message
27
+ var msg MockMessage
29
28
  if err := r.Decode(&msg); err == io.EOF {
30
29
  break
31
30
  }
@@ -47,7 +46,7 @@ var _ = Describe("Consumer", func() {
47
46
  Expect(subject.LastSync()).To(BeTemporally("~", time.Now(), time.Second))
48
47
  Expect(subject.LastModified()).To(BeTemporally("~", time.Unix(1515151515, 0), time.Second))
49
48
  Expect(subject.NumRead()).To(Equal(2))
50
- Expect(subject.Data()).To(Equal([]tbp.Message{fixture, fixture}))
49
+ Expect(subject.Data()).To(Equal([]MockMessage{fixture, fixture}))
51
50
  Expect(subject.Close()).To(Succeed())
52
51
  })
53
52
  })
data/feedx.gemspec CHANGED
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'feedx'
3
- s.version = '0.8.0'
3
+ s.version = '0.10.2'
4
4
  s.authors = ['Black Square Media Ltd']
5
5
  s.email = ['info@blacksquaremedia.com']
6
6
  s.summary = %(Exchange data between components via feeds)
@@ -8,7 +8,7 @@ Gem::Specification.new do |s|
8
8
  s.homepage = 'https://github.com/bsm/feedx'
9
9
  s.license = 'Apache-2.0'
10
10
 
11
- s.files = `git ls-files -z`.split("\x0").reject {|f| f.match(%r{^spec/}) }
11
+ s.files = `git ls-files -z`.split("\x0").reject {|f| f.start_with?('spec/') }
12
12
  s.test_files = `git ls-files -z -- spec/*`.split("\x0")
13
13
  s.require_paths = ['lib']
14
14
  s.required_ruby_version = '>= 2.4'
data/feedx_test.go CHANGED
@@ -8,13 +8,11 @@ import (
8
8
 
9
9
  "github.com/bsm/bfs"
10
10
  "github.com/bsm/feedx"
11
- tbp "github.com/golang/protobuf/proto/proto3_proto"
11
+ "github.com/gogo/protobuf/proto"
12
12
  . "github.com/onsi/ginkgo"
13
13
  . "github.com/onsi/gomega"
14
14
  )
15
15
 
16
- // ------------------------------------------------------------------------
17
-
18
16
  var memStore *bfs.InMem
19
17
 
20
18
  func init() {
@@ -24,19 +22,37 @@ func init() {
24
22
  })
25
23
  }
26
24
 
27
- var fixture = tbp.Message{
28
- Name: "Joe",
29
- Hilarity: tbp.Message_BILL_BAILEY,
30
- HeightInCm: 180,
25
+ // ------------------------------------------------------------------------
26
+
27
+ type Mock_Enum int32
28
+
29
+ const (
30
+ Mock_UNKNOWN Mock_Enum = 0
31
+ Mock_FIRST Mock_Enum = 3
32
+ )
33
+
34
+ type MockMessage struct {
35
+ Name string `protobuf:"bytes,1,opt,name=name,proto3"`
36
+ Enum Mock_Enum `protobuf:"varint,2,opt,name=enum,proto3"`
37
+ Height uint32 `protobuf:"varint,3,opt,name=height"`
31
38
  }
32
39
 
40
+ func (m *MockMessage) Reset() { *m = MockMessage{} }
41
+ func (m *MockMessage) String() string { return proto.CompactTextString(m) }
42
+ func (*MockMessage) ProtoMessage() {}
43
+
44
+ var fixture = MockMessage{
45
+ Name: "Joe",
46
+ Enum: Mock_FIRST,
47
+ Height: 180,
48
+ }
49
+
50
+ // ------------------------------------------------------------------------
51
+
33
52
  func writeMulti(obj *bfs.Object, numEntries int) error {
34
- w, err := feedx.NewWriter(context.Background(), obj, &feedx.WriterOptions{
53
+ w := feedx.NewWriter(context.Background(), obj, &feedx.WriterOptions{
35
54
  LastMod: time.Unix(1515151515, 123456789),
36
55
  })
37
- if err != nil {
38
- return err
39
- }
40
56
  defer w.Discard()
41
57
 
42
58
  for i := 0; i < numEntries; i++ {
data/format_test.go CHANGED
@@ -5,7 +5,6 @@ import (
5
5
  "io"
6
6
 
7
7
  "github.com/bsm/feedx"
8
- tbp "github.com/golang/protobuf/proto/proto3_proto"
9
8
  . "github.com/onsi/ginkgo"
10
9
  . "github.com/onsi/gomega"
11
10
  )
@@ -27,15 +26,15 @@ var _ = Describe("Format", func() {
27
26
  Expect(err).NotTo(HaveOccurred())
28
27
  defer dec.Close()
29
28
 
30
- v1 := new(tbp.Message)
29
+ v1 := new(MockMessage)
31
30
  Expect(dec.Decode(v1)).To(Succeed())
32
31
  Expect(v1.Name).To(Equal("Joe"))
33
32
 
34
- v2 := new(tbp.Message)
33
+ v2 := new(MockMessage)
35
34
  Expect(dec.Decode(v2)).To(Succeed())
36
35
  Expect(v2.Name).To(Equal("Joe"))
37
36
 
38
- v3 := new(tbp.Message)
37
+ v3 := new(MockMessage)
39
38
  Expect(dec.Decode(v3)).To(MatchError(io.EOF))
40
39
 
41
40
  Expect(dec.Close()).To(Succeed())
data/go.mod CHANGED
@@ -1,9 +1,10 @@
1
1
  module github.com/bsm/feedx
2
2
 
3
3
  require (
4
- github.com/bsm/bfs v0.8.0
5
- github.com/gogo/protobuf v1.3.0
6
- github.com/golang/protobuf v1.3.2
4
+ github.com/bmatcuk/doublestar v1.3.0 // indirect
5
+ github.com/bsm/bfs v0.10.1
6
+ github.com/gogo/protobuf v1.3.1
7
+ github.com/golang/protobuf v1.4.0
7
8
  github.com/onsi/ginkgo v1.10.2
8
9
  github.com/onsi/gomega v1.7.0
9
10
  golang.org/x/net v0.0.0-20191007182048-72f939374954 // indirect
data/go.sum CHANGED
@@ -1,15 +1,27 @@
1
- github.com/bmatcuk/doublestar v1.1.5 h1:2bNwBOmhyFEFcoB3tGvTD5xanq+4kyOZlB8wFYbMjkk=
2
- github.com/bmatcuk/doublestar v1.1.5/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
3
- github.com/bsm/bfs v0.8.0 h1:/suMKytZ3WhVY62osdFm6VD+gJtxaohlUBuf76vGC78=
4
- github.com/bsm/bfs v0.8.0/go.mod h1:cVv0jyqUY/jbHoG/WYPuWvOaOhW/HZ4jl7/JMlypvAE=
1
+ github.com/bmatcuk/doublestar v1.2.2 h1:oC24CykoSAB8zd7XgruHo33E0cHJf/WhQA/7BeXj+x0=
2
+ github.com/bmatcuk/doublestar v1.2.2/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
3
+ github.com/bmatcuk/doublestar v1.3.0 h1:1jLE2y0VpSrOn/QR9G4f2RmrCtkM3AuATcWradjHUvM=
4
+ github.com/bmatcuk/doublestar v1.3.0/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
5
+ github.com/bsm/bfs v0.10.1 h1:npAGbpWvcL9Zvhe7FATY7DGJX2jQYlkGLnw3QhotvBc=
6
+ github.com/bsm/bfs v0.10.1/go.mod h1:N3md8kQvlteRDcfc8tqw759yW98dhj+6seWEVcg4CmM=
5
7
  github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
6
8
  github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
7
- github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE=
8
- github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
9
+ github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
10
+ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
9
11
  github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
10
12
  github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
11
13
  github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
12
14
  github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
15
+ github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
16
+ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
17
+ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
18
+ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
19
+ github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ=
20
+ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
21
+ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
22
+ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
23
+ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
24
+ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
13
25
  github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
14
26
  github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
15
27
  github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
@@ -46,6 +58,14 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
46
58
  golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
47
59
  golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
48
60
  golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
61
+ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
62
+ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
63
+ google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
64
+ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
65
+ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
66
+ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
67
+ google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw=
68
+ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
49
69
  gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
50
70
  gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
51
71
  gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
data/lib/feedx/compression/gzip.rb CHANGED
@@ -1,11 +1,25 @@
1
1
  require 'zlib'
2
2
 
3
3
  class Feedx::Compression::Gzip < Feedx::Compression::Abstract
4
- def self.reader(io, &block)
5
- Zlib::GzipReader.wrap(io, &block)
6
- end
4
+ class << self
5
+ def reader(io, &block)
6
+ force_binmode(io)
7
+ Zlib::GzipReader.wrap(io, &block)
8
+ end
9
+
10
+ def writer(io, &block)
11
+ force_binmode(io)
12
+ Zlib::GzipWriter.wrap(io, &block)
13
+ end
14
+
15
+ private
7
16
 
8
- def self.writer(io, &block)
9
- Zlib::GzipWriter.wrap(io, &block)
17
+ def force_binmode(io)
18
+ if io.respond_to?(:binmode)
19
+ io.binmode
20
+ elsif io.respond_to?(:set_encoding)
21
+ io.set_encoding(Encoding::BINARY)
22
+ end
23
+ end
10
24
  end
11
25
  end
data/lib/feedx/consumer.rb CHANGED
@@ -8,8 +8,8 @@ module Feedx
8
8
  include Enumerable
9
9
 
10
10
  # See constructor.
11
- def self.each(url, klass, opts={}, &block)
12
- new(url, klass, opts).each(&block)
11
+ def self.each(url, klass, **opts, &block)
12
+ new(url, klass, **opts).each(&block)
13
13
  end
14
14
 
15
15
  # @param [String] url the destination URL.
@@ -19,9 +19,9 @@ module Feedx
19
19
  # @option opts [Hash] :format_options format decode options. Default: {}.
20
20
  # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
21
21
  # @option opts [Feedx::Cache::Value] :cache cache value to store remote last modified time and consume conditionally.
22
- def initialize(url, klass, opts={})
22
+ def initialize(url, klass, **opts)
23
23
  @klass = klass
24
- @stream = Feedx::Stream.new(url, opts)
24
+ @stream = Feedx::Stream.new(url, **opts)
25
25
  @fmt_opts = opts[:format_options] || {}
26
26
  @cache = opts[:cache]
27
27
  end
data/lib/feedx/producer.rb CHANGED
@@ -6,8 +6,8 @@ module Feedx
6
6
  # Produces a relation as an encoded feed to a remote location.
7
7
  class Producer
8
8
  # See constructor.
9
- def self.perform(url, opts={}, &block)
10
- new(url, opts, &block).perform
9
+ def self.perform(url, **opts, &block)
10
+ new(url, **opts, &block).perform
11
11
  end
12
12
 
13
13
  # @param [String] url the destination URL.
@@ -19,11 +19,11 @@ module Feedx
19
19
  # @option opts [Time,Proc] :last_modified the last modified time, used to determine if a push is necessary.
20
20
  # @yield A block factory to generate the relation or enumerator.
21
21
  # @yieldreturn [Enumerable,ActiveRecord::Relation] the relation or enumerator to stream.
22
- def initialize(url, opts={}, &block)
22
+ def initialize(url, **opts, &block)
23
23
  @enum = opts[:enum] || block
24
24
  raise ArgumentError, "#{self.class.name}.new expects an :enum option or a block factory" unless @enum
25
25
 
26
- @stream = Feedx::Stream.new(url, opts)
26
+ @stream = Feedx::Stream.new(url, **opts)
27
27
  @last_mod = opts[:last_modified]
28
28
  @fmt_opts = opts[:format_options] || {}
29
29
  end
@@ -37,7 +37,8 @@ module Feedx
37
37
  metadata = @stream.blob.info.metadata
38
38
  remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
39
39
  return -1 unless local_rev > remote_rev
40
- rescue BFS::FileNotFound # rubocop:disable Lint/HandleExceptions
40
+ rescue BFS::FileNotFound
41
+ nil
41
42
  end if local_rev.positive?
42
43
 
43
44
  @stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|
data/lib/feedx/stream.rb CHANGED
@@ -10,7 +10,7 @@ module Feedx
10
10
  # @param [Hash] opts options
11
11
  # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
12
12
  # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
13
- def initialize(url, opts={})
13
+ def initialize(url, **opts)
14
14
  @blob = BFS::Blob.new(url)
15
15
  @format = detect_format(opts[:format])
16
16
  @compress = detect_compress(opts[:compress])
@@ -20,8 +20,8 @@ module Feedx
20
20
  # @param [Hash] opts BFS::Blob#open options
21
21
  # @yield A block over a formatted stream.
22
22
  # @yieldparam [Feedx::Format::Abstract] formatted input stream.
23
- def open(opts={})
24
- @blob.open(opts) do |io|
23
+ def open(**opts)
24
+ @blob.open(**opts) do |io|
25
25
  @compress.reader(io) do |cio|
26
26
  fmt = @format.new(cio)
27
27
  yield fmt
@@ -33,8 +33,8 @@ module Feedx
33
33
  # @param [Hash] opts BFS::Blob#create options
34
34
  # @yield A block over a formatted stream.
35
35
  # @yieldparam [Feedx::Format::Abstract] formatted output stream.
36
- def create(opts={})
37
- @blob.create(opts) do |io|
36
+ def create(**opts)
37
+ @blob.create(**opts) do |io|
38
38
  @compress.writer(io) do |cio|
39
39
  fmt = @format.new(cio)
40
40
  yield fmt
data/producer.go CHANGED
@@ -140,21 +140,14 @@ func (p *Producer) push() (*ProducerPush, error) {
140
140
  wopt.LastMod = modTime
141
141
  }
142
142
 
143
- // retrieve original last modified time
144
- lastMod, err := remoteLastModified(p.ctx, p.remote)
145
- if err != nil {
143
+ // retrieve original last modified time, skip if not modified
144
+ if rts, err := remoteLastModified(p.ctx, p.remote); err != nil {
146
145
  return nil, err
147
- }
148
-
149
- // skip push if not modified
150
- if lastMod.Time().Equal(wopt.LastMod) {
146
+ } else if rts == timestampFromTime(wopt.LastMod) {
151
147
  return &ProducerPush{Producer: p}, nil
152
148
  }
153
149
 
154
- writer, err := NewWriter(p.ctx, p.remote, &wopt)
155
- if err != nil {
156
- return nil, err
157
- }
150
+ writer := NewWriter(p.ctx, p.remote, &wopt)
158
151
  defer writer.Discard()
159
152
 
160
153
  if err := p.pfn(writer); err != nil {
data/producer_test.go CHANGED
@@ -2,6 +2,7 @@ package feedx_test
2
2
 
3
3
  import (
4
4
  "context"
5
+ "sync/atomic"
5
6
  "time"
6
7
 
7
8
  "github.com/bsm/bfs"
@@ -13,11 +14,14 @@ import (
13
14
  var _ = Describe("Producer", func() {
14
15
  var subject *feedx.Producer
15
16
  var obj *bfs.Object
17
+ var numRuns uint32
16
18
  var ctx = context.Background()
17
19
 
18
20
  setup := func(o *feedx.ProducerOptions) {
19
21
  var err error
20
22
  subject, err = feedx.NewProducerForRemote(ctx, obj, o, func(w *feedx.Writer) error {
23
+ atomic.AddUint32(&numRuns, 1)
24
+
21
25
  for i := 0; i < 10; i++ {
22
26
  fix := fixture
23
27
  if err := w.Encode(&fix); err != nil {
@@ -30,6 +34,7 @@ var _ = Describe("Producer", func() {
30
34
  }
31
35
 
32
36
  BeforeEach(func() {
37
+ atomic.StoreUint32(&numRuns, 0)
33
38
  obj = bfs.NewInMemObject("path/to/file.jsonz")
34
39
  })
35
40
 
@@ -54,16 +59,22 @@ var _ = Describe("Producer", func() {
54
59
 
55
60
  It("should produce with custom last-mod check", func() {
56
61
  setup(&feedx.ProducerOptions{
57
- LastModCheck: func(_ context.Context) (time.Time, error) { return time.Unix(1515151515, 0), nil },
62
+ Interval: 50 * time.Millisecond,
63
+ LastModCheck: func(_ context.Context) (time.Time, error) { return time.Unix(1515151515, 987654321), nil },
58
64
  })
59
65
 
60
- Expect(subject.LastPush()).To(BeTemporally("~", time.Now(), time.Second))
61
- Expect(subject.LastModified()).To(Equal(time.Unix(1515151515, 0)))
66
+ firstPush := subject.LastPush()
67
+ Expect(firstPush).To(BeTemporally("~", time.Now(), time.Second))
68
+ Expect(subject.LastModified()).To(Equal(time.Unix(1515151515, 987000000)))
62
69
  Expect(subject.NumWritten()).To(Equal(10))
70
+ Expect(atomic.LoadUint32(&numRuns)).To(Equal(uint32(1)))
63
71
 
64
72
  info, err := obj.Head(ctx)
65
73
  Expect(err).NotTo(HaveOccurred())
66
74
  Expect(info.Size).To(BeNumerically("~", 75, 10))
67
- Expect(info.Metadata).To(HaveKeyWithValue("X-Feedx-Last-Modified", "1515151515000"))
75
+ Expect(info.Metadata).To(HaveKeyWithValue("X-Feedx-Last-Modified", "1515151515987"))
76
+
77
+ Eventually(func() bool { return subject.LastPush().After(firstPush) }).Should(BeTrue())
78
+ Expect(atomic.LoadUint32(&numRuns)).To(Equal(uint32(1)))
68
79
  })
69
80
  })
data/reader_test.go CHANGED
@@ -8,7 +8,6 @@ import (
8
8
  "github.com/bsm/feedx"
9
9
 
10
10
  "github.com/bsm/bfs"
11
- tbp "github.com/golang/protobuf/proto/proto3_proto"
12
11
  . "github.com/onsi/ginkgo"
13
12
  . "github.com/onsi/gomega"
14
13
  )
@@ -34,14 +33,14 @@ var _ = Describe("Reader", func() {
34
33
  It("should read", func() {
35
34
  data, err := ioutil.ReadAll(subject)
36
35
  Expect(err).NotTo(HaveOccurred())
37
- Expect(len(data)).To(BeNumerically("~", 140, 20))
36
+ Expect(len(data)).To(BeNumerically("~", 110, 20))
38
37
  Expect(subject.NumRead()).To(Equal(0))
39
38
  })
40
39
 
41
40
  It("should decode", func() {
42
- var msgs []tbp.Message
41
+ var msgs []MockMessage
43
42
  for {
44
- var msg tbp.Message
43
+ var msg MockMessage
45
44
  err := subject.Decode(&msg)
46
45
  if err == io.EOF {
47
46
  break
@@ -50,7 +49,7 @@ var _ = Describe("Reader", func() {
50
49
  msgs = append(msgs, msg)
51
50
  }
52
51
 
53
- Expect(msgs).To(Equal([]tbp.Message{fixture, fixture, fixture}))
52
+ Expect(msgs).To(Equal([]MockMessage{fixture, fixture, fixture}))
54
53
  Expect(subject.NumRead()).To(Equal(3))
55
54
  })
56
55
  })
data/spec/feedx/compression/gzip_spec.rb CHANGED
@@ -5,11 +5,13 @@ RSpec.describe Feedx::Compression::Gzip do
5
5
  wio = StringIO.new
6
6
  described_class.writer(wio) {|w| w.write 'xyz' * 1000 }
7
7
  expect(wio.size).to be_within(20).of(40)
8
+ expect(wio.string.encoding).to eq(Encoding::BINARY)
8
9
 
9
10
  data = ''
10
11
  StringIO.open(wio.string) do |rio|
11
12
  described_class.reader(rio) {|z| data = z.read }
12
13
  end
13
14
  expect(data.size).to eq(3000)
15
+ expect(data.encoding).to eq(Encoding.default_external)
14
16
  end
15
17
  end
data/spec/feedx/consumer_spec.rb CHANGED
@@ -36,10 +36,13 @@ RSpec.describe Feedx::Consumer do
36
36
 
37
37
  private
38
38
 
39
- def mock_produce!(opts={})
39
+ def mock_produce!(enum: mock_enum, **opts)
40
40
  url = 'mock:///dir/file.json'
41
- opts[:enum] ||= %w[x y z].map {|t| Feedx::TestCase::Model.new(t) } * 100
42
- Feedx::Producer.perform url, opts
41
+ Feedx::Producer.perform url, enum: enum, **opts
43
42
  url
44
43
  end
44
+
45
+ def mock_enum
46
+ %w[x y z].map {|t| Feedx::TestCase::Model.new(t) } * 100
47
+ end
45
48
  end
data/writer.go CHANGED
@@ -39,33 +39,29 @@ func (o *WriterOptions) norm(name string) {
39
39
  // Writer encodes feeds to remote locations.
40
40
  type Writer struct {
41
41
  ctx context.Context
42
- cancel context.CancelFunc
43
-
44
42
  remote *bfs.Object
45
43
  opt WriterOptions
46
44
  num int
47
45
 
48
- bw io.WriteCloser // bfs writer
46
+ bw bfs.Writer
49
47
  cw io.WriteCloser // compression writer
50
48
  ww *bufio.Writer
51
49
  fe FormatEncoder
52
50
  }
53
51
 
54
52
  // NewWriter inits a new feed writer.
55
- func NewWriter(ctx context.Context, remote *bfs.Object, opt *WriterOptions) (*Writer, error) {
53
+ func NewWriter(ctx context.Context, remote *bfs.Object, opt *WriterOptions) *Writer {
56
54
  var o WriterOptions
57
55
  if opt != nil {
58
56
  o = *opt
59
57
  }
60
58
  o.norm(remote.Name())
61
59
 
62
- ctx, cancel := context.WithCancel(ctx)
63
60
  return &Writer{
64
61
  ctx: ctx,
65
- cancel: cancel,
66
62
  remote: remote,
67
63
  opt: o,
68
- }, nil
64
+ }
69
65
  }
70
66
 
71
67
  // Write write raw bytes to the feed.
@@ -113,13 +109,27 @@ func (w *Writer) NumWritten() int {
113
109
 
114
110
  // Discard closes the writer and discards the contents.
115
111
  func (w *Writer) Discard() error {
116
- w.cancel()
117
- return w.Commit()
112
+ err := w.close()
113
+ if w.bw != nil {
114
+ if e := w.bw.Discard(); e != nil {
115
+ err = e
116
+ }
117
+ }
118
+ return err
118
119
  }
119
120
 
120
121
  // Commit closes the writer and persists the contents.
121
122
  func (w *Writer) Commit() error {
122
- var err error
123
+ err := w.close()
124
+ if w.bw != nil {
125
+ if e := w.bw.Commit(); e != nil {
126
+ err = e
127
+ }
128
+ }
129
+ return err
130
+ }
131
+
132
+ func (w *Writer) close() (err error) {
123
133
  if w.fe != nil {
124
134
  if e := w.fe.Close(); e != nil {
125
135
  err = e
@@ -135,11 +145,6 @@ func (w *Writer) Commit() error {
135
145
  err = e
136
146
  }
137
147
  }
138
- if w.bw != nil {
139
- if e := w.bw.Close(); e != nil {
140
- err = e
141
- }
142
- }
143
148
  return err
144
149
  }
145
150
 
data/writer_test.go CHANGED
@@ -21,10 +21,9 @@ var _ = Describe("Writer", func() {
21
21
  })
22
22
 
23
23
  It("should write plain", func() {
24
- w, err := feedx.NewWriter(context.Background(), plain, &feedx.WriterOptions{
24
+ w := feedx.NewWriter(context.Background(), plain, &feedx.WriterOptions{
25
25
  LastMod: time.Unix(1515151515, 123456789),
26
26
  })
27
- Expect(err).NotTo(HaveOccurred())
28
27
  defer w.Discard()
29
28
 
30
29
  Expect(w.Write(bytes.Repeat([]byte{'x'}, 10000))).To(Equal(10000))
@@ -37,10 +36,9 @@ var _ = Describe("Writer", func() {
37
36
  })
38
37
 
39
38
  It("should write compressed", func() {
40
- w, err := feedx.NewWriter(context.Background(), compressed, &feedx.WriterOptions{
39
+ w := feedx.NewWriter(context.Background(), compressed, &feedx.WriterOptions{
41
40
  LastMod: time.Unix(1515151515, 123456789),
42
41
  })
43
- Expect(err).NotTo(HaveOccurred())
44
42
  defer w.Discard()
45
43
 
46
44
  Expect(w.Write(bytes.Repeat([]byte{'x'}, 10000))).To(Equal(10000))
@@ -58,7 +56,7 @@ var _ = Describe("Writer", func() {
58
56
 
59
57
  info, err := plain.Head(ctx)
60
58
  Expect(err).NotTo(HaveOccurred())
61
- Expect(info.Size).To(BeNumerically("~", 470, 10))
59
+ Expect(info.Size).To(BeNumerically("~", 370, 10))
62
60
  Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
63
61
 
64
62
  info, err = compressed.Head(ctx)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: feedx
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.0
4
+ version: 0.10.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Black Square Media Ltd
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-10-09 00:00:00.000000000 Z
11
+ date: 2020-05-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bfs
@@ -191,7 +191,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
191
191
  - !ruby/object:Gem::Version
192
192
  version: '0'
193
193
  requirements: []
194
- rubygems_version: 3.0.3
194
+ rubygems_version: 3.1.2
195
195
  signing_key:
196
196
  specification_version: 4
197
197
  summary: Exchange data between components via feeds