feedx 0.8.0 → 0.10.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (22) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/.travis.yml +4 -12
  4. data/Gemfile.lock +23 -20
  5. data/consumer_test.go +3 -4
  6. data/feedx.gemspec +2 -2
  7. data/feedx_test.go +27 -11
  8. data/format_test.go +3 -4
  9. data/go.mod +4 -3
  10. data/go.sum +26 -6
  11. data/lib/feedx/compression/gzip.rb +19 -5
  12. data/lib/feedx/consumer.rb +4 -4
  13. data/lib/feedx/producer.rb +6 -5
  14. data/lib/feedx/stream.rb +5 -5
  15. data/producer.go +4 -11
  16. data/producer_test.go +15 -4
  17. data/reader_test.go +4 -5
  18. data/spec/feedx/compression/gzip_spec.rb +2 -0
  19. data/spec/feedx/consumer_spec.rb +6 -3
  20. data/writer.go +20 -15
  21. data/writer_test.go +3 -5
  22. metadata +3 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 3d61930af3b031d191151820e5fefc106e3c64d8ece5d21513d8c91e4db195fb
4
- data.tar.gz: 005bf01fefeb8b33299063f9c4e132ee9f1237f19a8bbf35c907e78863474121
3
+ metadata.gz: 73577178d9531fd5397bca23cb9e85cb0d6ab8ea9f97e32b0ce5b97dbebe43d4
4
+ data.tar.gz: e1439be97329b54309831cd7c19c19982058e71879dbc612058f0bb5fce1a578
5
5
  SHA512:
6
- metadata.gz: a4027d850b2122075bbb4d3ddc776fcd1be28f2f5db3d22c57a348d76fd821630554fe3327ac486b47b9b83e72c590476a68ab12eebc373d0fac231bb7ce449f
7
- data.tar.gz: 857a3fc8cf13142a02634363e60478abd9171ef84c0d9a61f163c5debbea0a7a8378864d0e5ce27b4bdbe9a89d4b57df930c72a1da79602929a914a7ade38032
6
+ metadata.gz: 673e28b0f7d0e01543796f7aa5ea869a80e119c2fbbcbb21eb9d6918ea4e051ecc20e6f1634e79fcc56acf1991ae68b26f7f69e1ec43c47a8b96d8a359d20834
7
+ data.tar.gz: f2b9d69ef7dbc915c4a08c798e0da5f65a044907b27c3e742a2c4a7c1a1b5c514e648fc1924a335b99ca65d56454718b0a9041827306f6f19ce0a7c2d6507a97
data/.gitignore CHANGED
@@ -1,2 +1,3 @@
1
1
  .rubocop-*
2
2
  pkg/
3
+ *~
data/.travis.yml CHANGED
@@ -2,21 +2,13 @@ matrix:
2
2
  include:
3
3
  - language: ruby
4
4
  rvm:
5
- - 2.6
6
- before_install:
7
- - gem install bundler
5
+ - 2.7
8
6
  - language: ruby
9
7
  rvm:
10
- - 2.5
11
- before_install:
12
- - gem install bundler
8
+ - 2.6
13
9
  - language: go
14
10
  go:
15
- - 1.13.x
16
- env:
17
- - GO111MODULE=on
11
+ - 1.14.x
18
12
  - language: go
19
13
  go:
20
- - 1.12.x
21
- env:
22
- - GO111MODULE=on
14
+ - 1.13.x
data/Gemfile.lock CHANGED
@@ -1,48 +1,51 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- feedx (0.8.0)
4
+ feedx (0.10.2)
5
5
  bfs (>= 0.5.0)
6
6
 
7
7
  GEM
8
8
  remote: https://rubygems.org/
9
9
  specs:
10
10
  ast (2.4.0)
11
- bfs (0.5.0)
11
+ bfs (0.7.0)
12
12
  diff-lcs (1.3)
13
- google-protobuf (3.10.0-x86_64-linux)
14
- jaro_winkler (1.5.3)
15
- parallel (1.18.0)
16
- parser (2.6.5.0)
13
+ google-protobuf (3.12.2)
14
+ parallel (1.19.1)
15
+ parser (2.7.1.3)
17
16
  ast (~> 2.4.0)
18
- pbio (0.1.0)
17
+ pbio (0.2.1)
19
18
  google-protobuf
20
19
  rainbow (3.0.0)
21
- rake (13.0.0)
20
+ rake (13.0.1)
21
+ rexml (3.2.4)
22
22
  rspec (3.9.0)
23
23
  rspec-core (~> 3.9.0)
24
24
  rspec-expectations (~> 3.9.0)
25
25
  rspec-mocks (~> 3.9.0)
26
- rspec-core (3.9.0)
27
- rspec-support (~> 3.9.0)
28
- rspec-expectations (3.9.0)
26
+ rspec-core (3.9.2)
27
+ rspec-support (~> 3.9.3)
28
+ rspec-expectations (3.9.2)
29
29
  diff-lcs (>= 1.2.0, < 2.0)
30
30
  rspec-support (~> 3.9.0)
31
- rspec-mocks (3.9.0)
31
+ rspec-mocks (3.9.1)
32
32
  diff-lcs (>= 1.2.0, < 2.0)
33
33
  rspec-support (~> 3.9.0)
34
- rspec-support (3.9.0)
35
- rubocop (0.75.0)
36
- jaro_winkler (~> 1.5.1)
34
+ rspec-support (3.9.3)
35
+ rubocop (0.84.0)
37
36
  parallel (~> 1.10)
38
- parser (>= 2.6)
37
+ parser (>= 2.7.0.1)
39
38
  rainbow (>= 2.2.2, < 4.0)
39
+ rexml
40
+ rubocop-ast (>= 0.0.3)
40
41
  ruby-progressbar (~> 1.7)
41
- unicode-display_width (>= 1.4.0, < 1.7)
42
- rubocop-performance (1.5.0)
42
+ unicode-display_width (>= 1.4.0, < 2.0)
43
+ rubocop-ast (0.0.3)
44
+ parser (>= 2.7.0.1)
45
+ rubocop-performance (1.6.0)
43
46
  rubocop (>= 0.71.0)
44
47
  ruby-progressbar (1.10.1)
45
- unicode-display_width (1.6.0)
48
+ unicode-display_width (1.7.0)
46
49
 
47
50
  PLATFORMS
48
51
  ruby
@@ -58,4 +61,4 @@ DEPENDENCIES
58
61
  rubocop-performance
59
62
 
60
63
  BUNDLED WITH
61
- 2.0.2
64
+ 2.1.2
data/consumer_test.go CHANGED
@@ -7,7 +7,6 @@ import (
7
7
 
8
8
  "github.com/bsm/bfs"
9
9
  "github.com/bsm/feedx"
10
- tbp "github.com/golang/protobuf/proto/proto3_proto"
11
10
  . "github.com/onsi/ginkgo"
12
11
  . "github.com/onsi/gomega"
13
12
  )
@@ -23,9 +22,9 @@ var _ = Describe("Consumer", func() {
23
22
 
24
23
  var err error
25
24
  subject, err = feedx.NewConsumerForRemote(ctx, obj, nil, func(r *feedx.Reader) (interface{}, error) {
26
- var msgs []tbp.Message
25
+ var msgs []MockMessage
27
26
  for {
28
- var msg tbp.Message
27
+ var msg MockMessage
29
28
  if err := r.Decode(&msg); err == io.EOF {
30
29
  break
31
30
  }
@@ -47,7 +46,7 @@ var _ = Describe("Consumer", func() {
47
46
  Expect(subject.LastSync()).To(BeTemporally("~", time.Now(), time.Second))
48
47
  Expect(subject.LastModified()).To(BeTemporally("~", time.Unix(1515151515, 0), time.Second))
49
48
  Expect(subject.NumRead()).To(Equal(2))
50
- Expect(subject.Data()).To(Equal([]tbp.Message{fixture, fixture}))
49
+ Expect(subject.Data()).To(Equal([]MockMessage{fixture, fixture}))
51
50
  Expect(subject.Close()).To(Succeed())
52
51
  })
53
52
  })
data/feedx.gemspec CHANGED
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'feedx'
3
- s.version = '0.8.0'
3
+ s.version = '0.10.2'
4
4
  s.authors = ['Black Square Media Ltd']
5
5
  s.email = ['info@blacksquaremedia.com']
6
6
  s.summary = %(Exchange data between components via feeds)
@@ -8,7 +8,7 @@ Gem::Specification.new do |s|
8
8
  s.homepage = 'https://github.com/bsm/feedx'
9
9
  s.license = 'Apache-2.0'
10
10
 
11
- s.files = `git ls-files -z`.split("\x0").reject {|f| f.match(%r{^spec/}) }
11
+ s.files = `git ls-files -z`.split("\x0").reject {|f| f.start_with?('spec/') }
12
12
  s.test_files = `git ls-files -z -- spec/*`.split("\x0")
13
13
  s.require_paths = ['lib']
14
14
  s.required_ruby_version = '>= 2.4'
data/feedx_test.go CHANGED
@@ -8,13 +8,11 @@ import (
8
8
 
9
9
  "github.com/bsm/bfs"
10
10
  "github.com/bsm/feedx"
11
- tbp "github.com/golang/protobuf/proto/proto3_proto"
11
+ "github.com/gogo/protobuf/proto"
12
12
  . "github.com/onsi/ginkgo"
13
13
  . "github.com/onsi/gomega"
14
14
  )
15
15
 
16
- // ------------------------------------------------------------------------
17
-
18
16
  var memStore *bfs.InMem
19
17
 
20
18
  func init() {
@@ -24,19 +22,37 @@ func init() {
24
22
  })
25
23
  }
26
24
 
27
- var fixture = tbp.Message{
28
- Name: "Joe",
29
- Hilarity: tbp.Message_BILL_BAILEY,
30
- HeightInCm: 180,
25
+ // ------------------------------------------------------------------------
26
+
27
+ type Mock_Enum int32
28
+
29
+ const (
30
+ Mock_UNKNOWN Mock_Enum = 0
31
+ Mock_FIRST Mock_Enum = 3
32
+ )
33
+
34
+ type MockMessage struct {
35
+ Name string `protobuf:"bytes,1,opt,name=name,proto3"`
36
+ Enum Mock_Enum `protobuf:"varint,2,opt,name=enum,proto3"`
37
+ Height uint32 `protobuf:"varint,3,opt,name=height"`
31
38
  }
32
39
 
40
+ func (m *MockMessage) Reset() { *m = MockMessage{} }
41
+ func (m *MockMessage) String() string { return proto.CompactTextString(m) }
42
+ func (*MockMessage) ProtoMessage() {}
43
+
44
+ var fixture = MockMessage{
45
+ Name: "Joe",
46
+ Enum: Mock_FIRST,
47
+ Height: 180,
48
+ }
49
+
50
+ // ------------------------------------------------------------------------
51
+
33
52
  func writeMulti(obj *bfs.Object, numEntries int) error {
34
- w, err := feedx.NewWriter(context.Background(), obj, &feedx.WriterOptions{
53
+ w := feedx.NewWriter(context.Background(), obj, &feedx.WriterOptions{
35
54
  LastMod: time.Unix(1515151515, 123456789),
36
55
  })
37
- if err != nil {
38
- return err
39
- }
40
56
  defer w.Discard()
41
57
 
42
58
  for i := 0; i < numEntries; i++ {
data/format_test.go CHANGED
@@ -5,7 +5,6 @@ import (
5
5
  "io"
6
6
 
7
7
  "github.com/bsm/feedx"
8
- tbp "github.com/golang/protobuf/proto/proto3_proto"
9
8
  . "github.com/onsi/ginkgo"
10
9
  . "github.com/onsi/gomega"
11
10
  )
@@ -27,15 +26,15 @@ var _ = Describe("Format", func() {
27
26
  Expect(err).NotTo(HaveOccurred())
28
27
  defer dec.Close()
29
28
 
30
- v1 := new(tbp.Message)
29
+ v1 := new(MockMessage)
31
30
  Expect(dec.Decode(v1)).To(Succeed())
32
31
  Expect(v1.Name).To(Equal("Joe"))
33
32
 
34
- v2 := new(tbp.Message)
33
+ v2 := new(MockMessage)
35
34
  Expect(dec.Decode(v2)).To(Succeed())
36
35
  Expect(v2.Name).To(Equal("Joe"))
37
36
 
38
- v3 := new(tbp.Message)
37
+ v3 := new(MockMessage)
39
38
  Expect(dec.Decode(v3)).To(MatchError(io.EOF))
40
39
 
41
40
  Expect(dec.Close()).To(Succeed())
data/go.mod CHANGED
@@ -1,9 +1,10 @@
1
1
  module github.com/bsm/feedx
2
2
 
3
3
  require (
4
- github.com/bsm/bfs v0.8.0
5
- github.com/gogo/protobuf v1.3.0
6
- github.com/golang/protobuf v1.3.2
4
+ github.com/bmatcuk/doublestar v1.3.0 // indirect
5
+ github.com/bsm/bfs v0.10.1
6
+ github.com/gogo/protobuf v1.3.1
7
+ github.com/golang/protobuf v1.4.0
7
8
  github.com/onsi/ginkgo v1.10.2
8
9
  github.com/onsi/gomega v1.7.0
9
10
  golang.org/x/net v0.0.0-20191007182048-72f939374954 // indirect
data/go.sum CHANGED
@@ -1,15 +1,27 @@
1
- github.com/bmatcuk/doublestar v1.1.5 h1:2bNwBOmhyFEFcoB3tGvTD5xanq+4kyOZlB8wFYbMjkk=
2
- github.com/bmatcuk/doublestar v1.1.5/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
3
- github.com/bsm/bfs v0.8.0 h1:/suMKytZ3WhVY62osdFm6VD+gJtxaohlUBuf76vGC78=
4
- github.com/bsm/bfs v0.8.0/go.mod h1:cVv0jyqUY/jbHoG/WYPuWvOaOhW/HZ4jl7/JMlypvAE=
1
+ github.com/bmatcuk/doublestar v1.2.2 h1:oC24CykoSAB8zd7XgruHo33E0cHJf/WhQA/7BeXj+x0=
2
+ github.com/bmatcuk/doublestar v1.2.2/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
3
+ github.com/bmatcuk/doublestar v1.3.0 h1:1jLE2y0VpSrOn/QR9G4f2RmrCtkM3AuATcWradjHUvM=
4
+ github.com/bmatcuk/doublestar v1.3.0/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
5
+ github.com/bsm/bfs v0.10.1 h1:npAGbpWvcL9Zvhe7FATY7DGJX2jQYlkGLnw3QhotvBc=
6
+ github.com/bsm/bfs v0.10.1/go.mod h1:N3md8kQvlteRDcfc8tqw759yW98dhj+6seWEVcg4CmM=
5
7
  github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
6
8
  github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
7
- github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE=
8
- github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
9
+ github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
10
+ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
9
11
  github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
10
12
  github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
11
13
  github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
12
14
  github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
15
+ github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
16
+ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
17
+ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
18
+ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
19
+ github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ=
20
+ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
21
+ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
22
+ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
23
+ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
24
+ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
13
25
  github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
14
26
  github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
15
27
  github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
@@ -46,6 +58,14 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
46
58
  golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
47
59
  golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
48
60
  golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
61
+ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
62
+ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
63
+ google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
64
+ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
65
+ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
66
+ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
67
+ google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw=
68
+ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
49
69
  gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
50
70
  gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
51
71
  gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
data/lib/feedx/compression/gzip.rb CHANGED
@@ -1,11 +1,25 @@
1
1
  require 'zlib'
2
2
 
3
3
  class Feedx::Compression::Gzip < Feedx::Compression::Abstract
4
- def self.reader(io, &block)
5
- Zlib::GzipReader.wrap(io, &block)
6
- end
4
+ class << self
5
+ def reader(io, &block)
6
+ force_binmode(io)
7
+ Zlib::GzipReader.wrap(io, &block)
8
+ end
9
+
10
+ def writer(io, &block)
11
+ force_binmode(io)
12
+ Zlib::GzipWriter.wrap(io, &block)
13
+ end
14
+
15
+ private
7
16
 
8
- def self.writer(io, &block)
9
- Zlib::GzipWriter.wrap(io, &block)
17
+ def force_binmode(io)
18
+ if io.respond_to?(:binmode)
19
+ io.binmode
20
+ elsif io.respond_to?(:set_encoding)
21
+ io.set_encoding(Encoding::BINARY)
22
+ end
23
+ end
10
24
  end
11
25
  end
data/lib/feedx/consumer.rb CHANGED
@@ -8,8 +8,8 @@ module Feedx
8
8
  include Enumerable
9
9
 
10
10
  # See constructor.
11
- def self.each(url, klass, opts={}, &block)
12
- new(url, klass, opts).each(&block)
11
+ def self.each(url, klass, **opts, &block)
12
+ new(url, klass, **opts).each(&block)
13
13
  end
14
14
 
15
15
  # @param [String] url the destination URL.
@@ -19,9 +19,9 @@ module Feedx
19
19
  # @option opts [Hash] :format_options format decode options. Default: {}.
20
20
  # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
21
21
  # @option opts [Feedx::Cache::Value] :cache cache value to store remote last modified time and consume conditionally.
22
- def initialize(url, klass, opts={})
22
+ def initialize(url, klass, **opts)
23
23
  @klass = klass
24
- @stream = Feedx::Stream.new(url, opts)
24
+ @stream = Feedx::Stream.new(url, **opts)
25
25
  @fmt_opts = opts[:format_options] || {}
26
26
  @cache = opts[:cache]
27
27
  end
data/lib/feedx/producer.rb CHANGED
@@ -6,8 +6,8 @@ module Feedx
6
6
  # Produces a relation as an encoded feed to a remote location.
7
7
  class Producer
8
8
  # See constructor.
9
- def self.perform(url, opts={}, &block)
10
- new(url, opts, &block).perform
9
+ def self.perform(url, **opts, &block)
10
+ new(url, **opts, &block).perform
11
11
  end
12
12
 
13
13
  # @param [String] url the destination URL.
@@ -19,11 +19,11 @@ module Feedx
19
19
  # @option opts [Time,Proc] :last_modified the last modified time, used to determine if a push is necessary.
20
20
  # @yield A block factory to generate the relation or enumerator.
21
21
  # @yieldreturn [Enumerable,ActiveRecord::Relation] the relation or enumerator to stream.
22
- def initialize(url, opts={}, &block)
22
+ def initialize(url, **opts, &block)
23
23
  @enum = opts[:enum] || block
24
24
  raise ArgumentError, "#{self.class.name}.new expects an :enum option or a block factory" unless @enum
25
25
 
26
- @stream = Feedx::Stream.new(url, opts)
26
+ @stream = Feedx::Stream.new(url, **opts)
27
27
  @last_mod = opts[:last_modified]
28
28
  @fmt_opts = opts[:format_options] || {}
29
29
  end
@@ -37,7 +37,8 @@ module Feedx
37
37
  metadata = @stream.blob.info.metadata
38
38
  remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
39
39
  return -1 unless local_rev > remote_rev
40
- rescue BFS::FileNotFound # rubocop:disable Lint/HandleExceptions
40
+ rescue BFS::FileNotFound
41
+ nil
41
42
  end if local_rev.positive?
42
43
 
43
44
  @stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|
data/lib/feedx/stream.rb CHANGED
@@ -10,7 +10,7 @@ module Feedx
10
10
  # @param [Hash] opts options
11
11
  # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
12
12
  # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
13
- def initialize(url, opts={})
13
+ def initialize(url, **opts)
14
14
  @blob = BFS::Blob.new(url)
15
15
  @format = detect_format(opts[:format])
16
16
  @compress = detect_compress(opts[:compress])
@@ -20,8 +20,8 @@ module Feedx
20
20
  # @param [Hash] opts BFS::Blob#open options
21
21
  # @yield A block over a formatted stream.
22
22
  # @yieldparam [Feedx::Format::Abstract] formatted input stream.
23
- def open(opts={})
24
- @blob.open(opts) do |io|
23
+ def open(**opts)
24
+ @blob.open(**opts) do |io|
25
25
  @compress.reader(io) do |cio|
26
26
  fmt = @format.new(cio)
27
27
  yield fmt
@@ -33,8 +33,8 @@ module Feedx
33
33
  # @param [Hash] opts BFS::Blob#create options
34
34
  # @yield A block over a formatted stream.
35
35
  # @yieldparam [Feedx::Format::Abstract] formatted output stream.
36
- def create(opts={})
37
- @blob.create(opts) do |io|
36
+ def create(**opts)
37
+ @blob.create(**opts) do |io|
38
38
  @compress.writer(io) do |cio|
39
39
  fmt = @format.new(cio)
40
40
  yield fmt
data/producer.go CHANGED
@@ -140,21 +140,14 @@ func (p *Producer) push() (*ProducerPush, error) {
140
140
  wopt.LastMod = modTime
141
141
  }
142
142
 
143
- // retrieve original last modified time
144
- lastMod, err := remoteLastModified(p.ctx, p.remote)
145
- if err != nil {
143
+ // retrieve original last modified time, skip if not modified
144
+ if rts, err := remoteLastModified(p.ctx, p.remote); err != nil {
146
145
  return nil, err
147
- }
148
-
149
- // skip push if not modified
150
- if lastMod.Time().Equal(wopt.LastMod) {
146
+ } else if rts == timestampFromTime(wopt.LastMod) {
151
147
  return &ProducerPush{Producer: p}, nil
152
148
  }
153
149
 
154
- writer, err := NewWriter(p.ctx, p.remote, &wopt)
155
- if err != nil {
156
- return nil, err
157
- }
150
+ writer := NewWriter(p.ctx, p.remote, &wopt)
158
151
  defer writer.Discard()
159
152
 
160
153
  if err := p.pfn(writer); err != nil {
data/producer_test.go CHANGED
@@ -2,6 +2,7 @@ package feedx_test
2
2
 
3
3
  import (
4
4
  "context"
5
+ "sync/atomic"
5
6
  "time"
6
7
 
7
8
  "github.com/bsm/bfs"
@@ -13,11 +14,14 @@ import (
13
14
  var _ = Describe("Producer", func() {
14
15
  var subject *feedx.Producer
15
16
  var obj *bfs.Object
17
+ var numRuns uint32
16
18
  var ctx = context.Background()
17
19
 
18
20
  setup := func(o *feedx.ProducerOptions) {
19
21
  var err error
20
22
  subject, err = feedx.NewProducerForRemote(ctx, obj, o, func(w *feedx.Writer) error {
23
+ atomic.AddUint32(&numRuns, 1)
24
+
21
25
  for i := 0; i < 10; i++ {
22
26
  fix := fixture
23
27
  if err := w.Encode(&fix); err != nil {
@@ -30,6 +34,7 @@ var _ = Describe("Producer", func() {
30
34
  }
31
35
 
32
36
  BeforeEach(func() {
37
+ atomic.StoreUint32(&numRuns, 0)
33
38
  obj = bfs.NewInMemObject("path/to/file.jsonz")
34
39
  })
35
40
 
@@ -54,16 +59,22 @@ var _ = Describe("Producer", func() {
54
59
 
55
60
  It("should produce with custom last-mod check", func() {
56
61
  setup(&feedx.ProducerOptions{
57
- LastModCheck: func(_ context.Context) (time.Time, error) { return time.Unix(1515151515, 0), nil },
62
+ Interval: 50 * time.Millisecond,
63
+ LastModCheck: func(_ context.Context) (time.Time, error) { return time.Unix(1515151515, 987654321), nil },
58
64
  })
59
65
 
60
- Expect(subject.LastPush()).To(BeTemporally("~", time.Now(), time.Second))
61
- Expect(subject.LastModified()).To(Equal(time.Unix(1515151515, 0)))
66
+ firstPush := subject.LastPush()
67
+ Expect(firstPush).To(BeTemporally("~", time.Now(), time.Second))
68
+ Expect(subject.LastModified()).To(Equal(time.Unix(1515151515, 987000000)))
62
69
  Expect(subject.NumWritten()).To(Equal(10))
70
+ Expect(atomic.LoadUint32(&numRuns)).To(Equal(uint32(1)))
63
71
 
64
72
  info, err := obj.Head(ctx)
65
73
  Expect(err).NotTo(HaveOccurred())
66
74
  Expect(info.Size).To(BeNumerically("~", 75, 10))
67
- Expect(info.Metadata).To(HaveKeyWithValue("X-Feedx-Last-Modified", "1515151515000"))
75
+ Expect(info.Metadata).To(HaveKeyWithValue("X-Feedx-Last-Modified", "1515151515987"))
76
+
77
+ Eventually(func() bool { return subject.LastPush().After(firstPush) }).Should(BeTrue())
78
+ Expect(atomic.LoadUint32(&numRuns)).To(Equal(uint32(1)))
68
79
  })
69
80
  })
data/reader_test.go CHANGED
@@ -8,7 +8,6 @@ import (
8
8
  "github.com/bsm/feedx"
9
9
 
10
10
  "github.com/bsm/bfs"
11
- tbp "github.com/golang/protobuf/proto/proto3_proto"
12
11
  . "github.com/onsi/ginkgo"
13
12
  . "github.com/onsi/gomega"
14
13
  )
@@ -34,14 +33,14 @@ var _ = Describe("Reader", func() {
34
33
  It("should read", func() {
35
34
  data, err := ioutil.ReadAll(subject)
36
35
  Expect(err).NotTo(HaveOccurred())
37
- Expect(len(data)).To(BeNumerically("~", 140, 20))
36
+ Expect(len(data)).To(BeNumerically("~", 110, 20))
38
37
  Expect(subject.NumRead()).To(Equal(0))
39
38
  })
40
39
 
41
40
  It("should decode", func() {
42
- var msgs []tbp.Message
41
+ var msgs []MockMessage
43
42
  for {
44
- var msg tbp.Message
43
+ var msg MockMessage
45
44
  err := subject.Decode(&msg)
46
45
  if err == io.EOF {
47
46
  break
@@ -50,7 +49,7 @@ var _ = Describe("Reader", func() {
50
49
  msgs = append(msgs, msg)
51
50
  }
52
51
 
53
- Expect(msgs).To(Equal([]tbp.Message{fixture, fixture, fixture}))
52
+ Expect(msgs).To(Equal([]MockMessage{fixture, fixture, fixture}))
54
53
  Expect(subject.NumRead()).To(Equal(3))
55
54
  })
56
55
  })
data/spec/feedx/compression/gzip_spec.rb CHANGED
@@ -5,11 +5,13 @@ RSpec.describe Feedx::Compression::Gzip do
5
5
  wio = StringIO.new
6
6
  described_class.writer(wio) {|w| w.write 'xyz' * 1000 }
7
7
  expect(wio.size).to be_within(20).of(40)
8
+ expect(wio.string.encoding).to eq(Encoding::BINARY)
8
9
 
9
10
  data = ''
10
11
  StringIO.open(wio.string) do |rio|
11
12
  described_class.reader(rio) {|z| data = z.read }
12
13
  end
13
14
  expect(data.size).to eq(3000)
15
+ expect(data.encoding).to eq(Encoding.default_external)
14
16
  end
15
17
  end
data/spec/feedx/consumer_spec.rb CHANGED
@@ -36,10 +36,13 @@ RSpec.describe Feedx::Consumer do
36
36
 
37
37
  private
38
38
 
39
- def mock_produce!(opts={})
39
+ def mock_produce!(enum: mock_enum, **opts)
40
40
  url = 'mock:///dir/file.json'
41
- opts[:enum] ||= %w[x y z].map {|t| Feedx::TestCase::Model.new(t) } * 100
42
- Feedx::Producer.perform url, opts
41
+ Feedx::Producer.perform url, enum: enum, **opts
43
42
  url
44
43
  end
44
+
45
+ def mock_enum
46
+ %w[x y z].map {|t| Feedx::TestCase::Model.new(t) } * 100
47
+ end
45
48
  end
data/writer.go CHANGED
@@ -39,33 +39,29 @@ func (o *WriterOptions) norm(name string) {
39
39
  // Writer encodes feeds to remote locations.
40
40
  type Writer struct {
41
41
  ctx context.Context
42
- cancel context.CancelFunc
43
-
44
42
  remote *bfs.Object
45
43
  opt WriterOptions
46
44
  num int
47
45
 
48
- bw io.WriteCloser // bfs writer
46
+ bw bfs.Writer
49
47
  cw io.WriteCloser // compression writer
50
48
  ww *bufio.Writer
51
49
  fe FormatEncoder
52
50
  }
53
51
 
54
52
  // NewWriter inits a new feed writer.
55
- func NewWriter(ctx context.Context, remote *bfs.Object, opt *WriterOptions) (*Writer, error) {
53
+ func NewWriter(ctx context.Context, remote *bfs.Object, opt *WriterOptions) *Writer {
56
54
  var o WriterOptions
57
55
  if opt != nil {
58
56
  o = *opt
59
57
  }
60
58
  o.norm(remote.Name())
61
59
 
62
- ctx, cancel := context.WithCancel(ctx)
63
60
  return &Writer{
64
61
  ctx: ctx,
65
- cancel: cancel,
66
62
  remote: remote,
67
63
  opt: o,
68
- }, nil
64
+ }
69
65
  }
70
66
 
71
67
  // Write write raw bytes to the feed.
@@ -113,13 +109,27 @@ func (w *Writer) NumWritten() int {
113
109
 
114
110
  // Discard closes the writer and discards the contents.
115
111
  func (w *Writer) Discard() error {
116
- w.cancel()
117
- return w.Commit()
112
+ err := w.close()
113
+ if w.bw != nil {
114
+ if e := w.bw.Discard(); e != nil {
115
+ err = e
116
+ }
117
+ }
118
+ return err
118
119
  }
119
120
 
120
121
  // Commit closes the writer and persists the contents.
121
122
  func (w *Writer) Commit() error {
122
- var err error
123
+ err := w.close()
124
+ if w.bw != nil {
125
+ if e := w.bw.Commit(); e != nil {
126
+ err = e
127
+ }
128
+ }
129
+ return err
130
+ }
131
+
132
+ func (w *Writer) close() (err error) {
123
133
  if w.fe != nil {
124
134
  if e := w.fe.Close(); e != nil {
125
135
  err = e
@@ -135,11 +145,6 @@ func (w *Writer) Commit() error {
135
145
  err = e
136
146
  }
137
147
  }
138
- if w.bw != nil {
139
- if e := w.bw.Close(); e != nil {
140
- err = e
141
- }
142
- }
143
148
  return err
144
149
  }
145
150
 
data/writer_test.go CHANGED
@@ -21,10 +21,9 @@ var _ = Describe("Writer", func() {
21
21
  })
22
22
 
23
23
  It("should write plain", func() {
24
- w, err := feedx.NewWriter(context.Background(), plain, &feedx.WriterOptions{
24
+ w := feedx.NewWriter(context.Background(), plain, &feedx.WriterOptions{
25
25
  LastMod: time.Unix(1515151515, 123456789),
26
26
  })
27
- Expect(err).NotTo(HaveOccurred())
28
27
  defer w.Discard()
29
28
 
30
29
  Expect(w.Write(bytes.Repeat([]byte{'x'}, 10000))).To(Equal(10000))
@@ -37,10 +36,9 @@ var _ = Describe("Writer", func() {
37
36
  })
38
37
 
39
38
  It("should write compressed", func() {
40
- w, err := feedx.NewWriter(context.Background(), compressed, &feedx.WriterOptions{
39
+ w := feedx.NewWriter(context.Background(), compressed, &feedx.WriterOptions{
41
40
  LastMod: time.Unix(1515151515, 123456789),
42
41
  })
43
- Expect(err).NotTo(HaveOccurred())
44
42
  defer w.Discard()
45
43
 
46
44
  Expect(w.Write(bytes.Repeat([]byte{'x'}, 10000))).To(Equal(10000))
@@ -58,7 +56,7 @@ var _ = Describe("Writer", func() {
58
56
 
59
57
  info, err := plain.Head(ctx)
60
58
  Expect(err).NotTo(HaveOccurred())
61
- Expect(info.Size).To(BeNumerically("~", 470, 10))
59
+ Expect(info.Size).To(BeNumerically("~", 370, 10))
62
60
  Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
63
61
 
64
62
  info, err = compressed.Head(ctx)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: feedx
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.0
4
+ version: 0.10.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Black Square Media Ltd
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-10-09 00:00:00.000000000 Z
11
+ date: 2020-05-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bfs
@@ -191,7 +191,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
191
191
  - !ruby/object:Gem::Version
192
192
  version: '0'
193
193
  requirements: []
194
- rubygems_version: 3.0.3
194
+ rubygems_version: 3.1.2
195
195
  signing_key:
196
196
  specification_version: 4
197
197
  summary: Exchange data between components via feeds