feedx 0.7.2 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c294d0811dec65d1fce8a44fe2e18dbe08ac0bffee8d56c6b1d96bdac63e9366
4
- data.tar.gz: e6172cbf1f728569624a928dcf2a22c3a9a87824ed6cfd8fddaf834d1e526d8e
3
+ metadata.gz: 3d61930af3b031d191151820e5fefc106e3c64d8ece5d21513d8c91e4db195fb
4
+ data.tar.gz: 005bf01fefeb8b33299063f9c4e132ee9f1237f19a8bbf35c907e78863474121
5
5
  SHA512:
6
- metadata.gz: 64f69d35aa2cdca873e223b0f959a404dacebe03b07a8ca77796a10d5d6dfc06b4f7b9e7ee3bbfcfa03c23656399038324088a19d8e643636d0d3f3408af7b4b
7
- data.tar.gz: fb2e106e250693c5756edecf6d98df7115f8e8b33f4fca01ade0c3c568eb756215e6efa967d36099fdb055e4f77a26f68aebc207d9f62bb82c4748fb193d8ec7
6
+ metadata.gz: a4027d850b2122075bbb4d3ddc776fcd1be28f2f5db3d22c57a348d76fd821630554fe3327ac486b47b9b83e72c590476a68ab12eebc373d0fac231bb7ce449f
7
+ data.tar.gz: 857a3fc8cf13142a02634363e60478abd9171ef84c0d9a61f163c5debbea0a7a8378864d0e5ce27b4bdbe9a89d4b57df930c72a1da79602929a914a7ade38032
@@ -10,18 +10,13 @@ matrix:
10
10
  - 2.5
11
11
  before_install:
12
12
  - gem install bundler
13
- - language: ruby
14
- rvm:
15
- - 2.4
16
- before_install:
17
- - gem install bundler
18
13
  - language: go
19
14
  go:
20
- - 1.12.x
15
+ - 1.13.x
21
16
  env:
22
17
  - GO111MODULE=on
23
18
  - language: go
24
19
  go:
25
- - 1.11.x
20
+ - 1.12.x
26
21
  env:
27
22
  - GO111MODULE=on
@@ -1,45 +1,45 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- feedx (0.7.2)
5
- bfs (>= 0.4.0)
4
+ feedx (0.8.0)
5
+ bfs (>= 0.5.0)
6
6
 
7
7
  GEM
8
8
  remote: https://rubygems.org/
9
9
  specs:
10
10
  ast (2.4.0)
11
- bfs (0.4.0)
11
+ bfs (0.5.0)
12
12
  diff-lcs (1.3)
13
- google-protobuf (3.8.0)
13
+ google-protobuf (3.10.0-x86_64-linux)
14
14
  jaro_winkler (1.5.3)
15
- parallel (1.17.0)
16
- parser (2.6.3.0)
15
+ parallel (1.18.0)
16
+ parser (2.6.5.0)
17
17
  ast (~> 2.4.0)
18
18
  pbio (0.1.0)
19
19
  google-protobuf
20
20
  rainbow (3.0.0)
21
- rake (12.3.2)
22
- rspec (3.8.0)
23
- rspec-core (~> 3.8.0)
24
- rspec-expectations (~> 3.8.0)
25
- rspec-mocks (~> 3.8.0)
26
- rspec-core (3.8.1)
27
- rspec-support (~> 3.8.0)
28
- rspec-expectations (3.8.4)
21
+ rake (13.0.0)
22
+ rspec (3.9.0)
23
+ rspec-core (~> 3.9.0)
24
+ rspec-expectations (~> 3.9.0)
25
+ rspec-mocks (~> 3.9.0)
26
+ rspec-core (3.9.0)
27
+ rspec-support (~> 3.9.0)
28
+ rspec-expectations (3.9.0)
29
29
  diff-lcs (>= 1.2.0, < 2.0)
30
- rspec-support (~> 3.8.0)
31
- rspec-mocks (3.8.1)
30
+ rspec-support (~> 3.9.0)
31
+ rspec-mocks (3.9.0)
32
32
  diff-lcs (>= 1.2.0, < 2.0)
33
- rspec-support (~> 3.8.0)
34
- rspec-support (3.8.2)
35
- rubocop (0.71.0)
33
+ rspec-support (~> 3.9.0)
34
+ rspec-support (3.9.0)
35
+ rubocop (0.75.0)
36
36
  jaro_winkler (~> 1.5.1)
37
37
  parallel (~> 1.10)
38
38
  parser (>= 2.6)
39
39
  rainbow (>= 2.2.2, < 4.0)
40
40
  ruby-progressbar (~> 1.7)
41
41
  unicode-display_width (>= 1.4.0, < 1.7)
42
- rubocop-performance (1.4.0)
42
+ rubocop-performance (1.5.0)
43
43
  rubocop (>= 0.71.0)
44
44
  ruby-progressbar (1.10.1)
45
45
  unicode-display_width (1.6.0)
@@ -58,4 +58,4 @@ DEPENDENCIES
58
58
  rubocop-performance
59
59
 
60
60
  BUNDLED WITH
61
- 2.0.1
61
+ 2.0.2
data/README.md CHANGED
@@ -17,7 +17,7 @@ producer = Feedx::Producer.new relation, 's3://my-bucket/feeds/users.json.gz'
17
17
 
18
18
  # Push a new feed every hour
19
19
  loop do
20
- pusher.perform
20
+ producer.perform
21
21
  sleep(3600)
22
22
  end
23
23
  ```
@@ -21,12 +21,11 @@ type ConsumerOptions struct {
21
21
  AfterSync func(*ConsumerSync, error)
22
22
  }
23
23
 
24
- func (o *ConsumerOptions) norm(name string) error {
24
+ func (o *ConsumerOptions) norm(name string) {
25
25
  o.ReaderOptions.norm(name)
26
26
  if o.Interval <= 0 {
27
27
  o.Interval = time.Minute
28
28
  }
29
- return nil
30
29
  }
31
30
 
32
31
  // ConsumerSync contains the state of the last sync.
@@ -42,7 +41,7 @@ type ConsumerSync struct {
42
41
  }
43
42
 
44
43
  // ConsumeFunc is a parsing callback which is run by the consumer every sync interval.
45
- type ConsumeFunc func(FormatDecoder) (data interface{}, err error)
44
+ type ConsumeFunc func(*Reader) (data interface{}, err error)
46
45
 
47
46
  // Consumer manages data retrieval from a remote feed.
48
47
  // It queries the feed in regular intervals, continuously retrieving new updates.
@@ -81,9 +80,7 @@ func NewConsumerForRemote(ctx context.Context, remote *bfs.Object, opt *Consumer
81
80
  if opt != nil {
82
81
  o = *opt
83
82
  }
84
- if err := o.norm(remote.Name()); err != nil {
85
- return nil, err
86
- }
83
+ o.norm(remote.Name())
87
84
 
88
85
  ctx, stop := context.WithCancel(ctx)
89
86
  c := &consumer{
@@ -22,11 +22,11 @@ var _ = Describe("Consumer", func() {
22
22
  Expect(writeMulti(obj, 2)).To(Succeed())
23
23
 
24
24
  var err error
25
- subject, err = feedx.NewConsumerForRemote(ctx, obj, nil, func(dec feedx.FormatDecoder) (interface{}, error) {
25
+ subject, err = feedx.NewConsumerForRemote(ctx, obj, nil, func(r *feedx.Reader) (interface{}, error) {
26
26
  var msgs []tbp.Message
27
27
  for {
28
28
  var msg tbp.Message
29
- if err := dec.Decode(&msg); err == io.EOF {
29
+ if err := r.Decode(&msg); err == io.EOF {
30
30
  break
31
31
  }
32
32
  if err != nil {
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'feedx'
3
- s.version = '0.7.2'
3
+ s.version = '0.8.0'
4
4
  s.authors = ['Black Square Media Ltd']
5
5
  s.email = ['info@blacksquaremedia.com']
6
6
  s.summary = %(Exchange data between components via feeds)
@@ -13,7 +13,7 @@ Gem::Specification.new do |s|
13
13
  s.require_paths = ['lib']
14
14
  s.required_ruby_version = '>= 2.4'
15
15
 
16
- s.add_dependency 'bfs', '>= 0.4.0'
16
+ s.add_dependency 'bfs', '>= 0.5.0'
17
17
 
18
18
  s.add_development_dependency 'bundler'
19
19
  s.add_development_dependency 'pbio'
data/feedx.go CHANGED
@@ -13,8 +13,8 @@ import (
13
13
  var ErrNotModified = errors.New("feedx: not modified")
14
14
 
15
15
  const (
16
- metaLastModified = "x-feedx-last-modified"
17
- metaPusherLastModified = "x-feedx-pusher-last-modified"
16
+ metaLastModified = "X-Feedx-Last-Modified"
17
+ metaPusherLastModified = "X-Feedx-Pusher-Last-Modified"
18
18
  )
19
19
 
20
20
  // Timestamp with millisecond resolution
@@ -35,9 +35,9 @@ func remoteLastModified(ctx context.Context, obj *bfs.Object) (timestamp, error)
35
35
  return 0, err
36
36
  }
37
37
 
38
- millis, _ := strconv.ParseInt(info.Metadata[metaLastModified], 10, 64)
38
+ millis, _ := strconv.ParseInt(info.Metadata.Get(metaLastModified), 10, 64)
39
39
  if millis == 0 {
40
- millis, _ = strconv.ParseInt(info.Metadata[metaPusherLastModified], 10, 64)
40
+ millis, _ = strconv.ParseInt(info.Metadata.Get(metaPusherLastModified), 10, 64)
41
41
  }
42
42
  return timestamp(millis), nil
43
43
  }
data/format.go CHANGED
@@ -2,6 +2,7 @@ package feedx
2
2
 
3
3
  import (
4
4
  "encoding/json"
5
+ "errors"
5
6
  "fmt"
6
7
  "io"
7
8
  "path"
@@ -11,6 +12,8 @@ import (
11
12
  pbio "github.com/gogo/protobuf/io"
12
13
  )
13
14
 
15
+ var errNoFormat = errors.New("feedx: no format detected")
16
+
14
17
  // Format represents the data format.
15
18
  type Format interface {
16
19
  // NewDecoder wraps a decoder around a reader.
@@ -36,7 +39,7 @@ func DetectFormat(name string) Format {
36
39
  return DetectFormat(name[0 : len(name)-len(ext)])
37
40
  }
38
41
  }
39
- return nil
42
+ return (*noFormat)(nil)
40
43
  }
41
44
 
42
45
  // FormatDecoder methods
@@ -47,20 +50,23 @@ type FormatDecoder interface {
47
50
  io.Closer
48
51
  }
49
52
 
50
- // FormatPureEncoder methods
51
- type FormatPureEncoder interface {
53
+ // FormatEncoder methods
54
+ type FormatEncoder interface {
52
55
  // Encode encodes the value to the stream.
53
56
  Encode(v interface{}) error
54
- }
55
57
 
56
- // FormatEncoder methods
57
- type FormatEncoder interface {
58
- FormatPureEncoder
59
58
  io.Closer
60
59
  }
61
60
 
62
61
  // --------------------------------------------------------------------
63
62
 
63
+ type noFormat struct{}
64
+
65
+ func (*noFormat) NewDecoder(r io.Reader) (FormatDecoder, error) { return nil, errNoFormat }
66
+ func (*noFormat) NewEncoder(w io.Writer) (FormatEncoder, error) { return nil, errNoFormat }
67
+
68
+ // --------------------------------------------------------------------
69
+
64
70
  // JSONFormat provides a Format implemention for JSON.
65
71
  var JSONFormat = jsonFormat{}
66
72
 
data/go.mod CHANGED
@@ -1,9 +1,15 @@
1
1
  module github.com/bsm/feedx
2
2
 
3
3
  require (
4
- github.com/bsm/bfs v0.6.3
5
- github.com/gogo/protobuf v1.2.1
6
- github.com/golang/protobuf v1.3.1
7
- github.com/onsi/ginkgo v1.8.0
8
- github.com/onsi/gomega v1.5.0
4
+ github.com/bsm/bfs v0.8.0
5
+ github.com/gogo/protobuf v1.3.0
6
+ github.com/golang/protobuf v1.3.2
7
+ github.com/onsi/ginkgo v1.10.2
8
+ github.com/onsi/gomega v1.7.0
9
+ golang.org/x/net v0.0.0-20191007182048-72f939374954 // indirect
10
+ golang.org/x/sys v0.0.0-20191008105621-543471e840be // indirect
11
+ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
12
+ gopkg.in/yaml.v2 v2.2.4 // indirect
9
13
  )
14
+
15
+ go 1.13
data/go.sum CHANGED
@@ -1,18 +1,18 @@
1
- github.com/bmatcuk/doublestar v1.1.1 h1:YroD6BJCZBYx06yYFEWvUuKVWQn3vLLQAVmDmvTSaiQ=
2
- github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
3
- github.com/bsm/bfs v0.6.3 h1:qHOkJIx5W971wlnFLeJfiGGO0TUdB9JpTJ9RTAVntVg=
4
- github.com/bsm/bfs v0.6.3/go.mod h1:bRNSGIGvKb0MlboBpb7Aa3/Kaxmhes/6Be6wD+fsnKc=
1
+ github.com/bmatcuk/doublestar v1.1.5 h1:2bNwBOmhyFEFcoB3tGvTD5xanq+4kyOZlB8wFYbMjkk=
2
+ github.com/bmatcuk/doublestar v1.1.5/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
3
+ github.com/bsm/bfs v0.8.0 h1:/suMKytZ3WhVY62osdFm6VD+gJtxaohlUBuf76vGC78=
4
+ github.com/bsm/bfs v0.8.0/go.mod h1:cVv0jyqUY/jbHoG/WYPuWvOaOhW/HZ4jl7/JMlypvAE=
5
5
  github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
6
6
  github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
7
- github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
8
- github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
7
+ github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE=
8
+ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
9
9
  github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
10
10
  github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
11
- github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
12
- github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
11
+ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
12
+ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
13
13
  github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
14
14
  github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
15
- github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
15
+ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
16
16
  github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
17
17
  github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
18
18
  github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -23,26 +23,34 @@ github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
23
23
  github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
24
24
  github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
25
25
  github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
26
+ github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94=
27
+ github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
26
28
  github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
27
29
  github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
30
+ github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
31
+ github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
28
32
  golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
29
33
  golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
30
- golang.org/x/net v0.0.0-20190514140710-3ec191127204 h1:4yG6GqBtw9C+UrLp6s2wtSniayy/Vd/3F7ffLE427XI=
31
- golang.org/x/net v0.0.0-20190514140710-3ec191127204/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
34
+ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
35
+ golang.org/x/net v0.0.0-20191007182048-72f939374954 h1:JGZucVF/L/TotR719NbujzadOZ2AgnYlqphQGHDCKaU=
36
+ golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
32
37
  golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
33
38
  golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
34
39
  golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
35
- golang.org/x/sys v0.0.0-20190516110030-61b9204099cb h1:k07iPOt0d6nEnwXF+kHB+iEg+WSuKe/SOQuFM2QoD+E=
36
- golang.org/x/sys v0.0.0-20190516110030-61b9204099cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
40
+ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
41
+ golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU=
42
+ golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
37
43
  golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
38
44
  golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
39
45
  golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
40
46
  golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
41
- golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
42
47
  golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
48
+ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
43
49
  gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
44
50
  gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
45
51
  gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
52
+ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
53
+ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
46
54
  gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
47
55
  gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
48
56
  gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
@@ -51,3 +59,5 @@ gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
51
59
  gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
52
60
  gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
53
61
  gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
62
+ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
63
+ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -1,5 +1,6 @@
1
1
  module Feedx
2
- META_LAST_MODIFIED = 'x-feedx-last-modified'.freeze
2
+ META_LAST_MODIFIED = 'X-Feedx-Last-Modified'.freeze
3
+ META_LAST_MODIFIED_DC = META_LAST_MODIFIED.downcase.freeze
3
4
 
4
5
  autoload :Cache, 'feedx/cache'
5
6
  autoload :Compression, 'feedx/compression'
@@ -31,8 +31,9 @@ module Feedx
31
31
  remote_rev = nil
32
32
 
33
33
  if @cache
34
+ metadata = @stream.blob.info.metadata
34
35
  local_rev = @cache.read.to_i
35
- remote_rev = @stream.blob.info.metadata[META_LAST_MODIFIED].to_i
36
+ remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
36
37
  return false if remote_rev.positive? && remote_rev <= local_rev
37
38
  end
38
39
 
@@ -34,7 +34,8 @@ module Feedx
34
34
  local_rev = last_mod.is_a?(Integer) ? last_mod : (last_mod.to_f * 1000).floor
35
35
 
36
36
  begin
37
- remote_rev = @stream.blob.info.metadata[META_LAST_MODIFIED].to_i
37
+ metadata = @stream.blob.info.metadata
38
+ remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
38
39
  return -1 unless local_rev > remote_rev
39
40
  rescue BFS::FileNotFound # rubocop:disable Lint/HandleExceptions
40
41
  end if local_rev.positive?
@@ -9,7 +9,7 @@ import (
9
9
  )
10
10
 
11
11
  // ProduceFunc is a callback which is run by the producer on every iteration.
12
- type ProduceFunc func(FormatPureEncoder) error
12
+ type ProduceFunc func(*Writer) error
13
13
 
14
14
  // ProducerOptions configure the producer instance.
15
15
  type ProducerOptions struct {
@@ -28,12 +28,11 @@ type ProducerOptions struct {
28
28
  AfterPush func(*ProducerPush, error)
29
29
  }
30
30
 
31
- func (o *ProducerOptions) norm(name string) error {
31
+ func (o *ProducerOptions) norm(name string) {
32
32
  o.WriterOptions.norm(name)
33
33
  if o.Interval <= 0 {
34
34
  o.Interval = time.Minute
35
35
  }
36
- return nil
37
36
  }
38
37
 
39
38
  // ProducerPush contains the state of the last push.
@@ -79,9 +78,7 @@ func NewProducerForRemote(ctx context.Context, remote *bfs.Object, opt *Producer
79
78
  if opt != nil {
80
79
  o = *opt
81
80
  }
82
- if err := o.norm(remote.Name()); err != nil {
83
- return nil, err
84
- }
81
+ o.norm(remote.Name())
85
82
 
86
83
  ctx, stop := context.WithCancel(ctx)
87
84
  p := &Producer{
@@ -17,10 +17,10 @@ var _ = Describe("Producer", func() {
17
17
 
18
18
  setup := func(o *feedx.ProducerOptions) {
19
19
  var err error
20
- subject, err = feedx.NewProducerForRemote(ctx, obj, o, func(enc feedx.FormatPureEncoder) error {
20
+ subject, err = feedx.NewProducerForRemote(ctx, obj, o, func(w *feedx.Writer) error {
21
21
  for i := 0; i < 10; i++ {
22
22
  fix := fixture
23
- if err := enc.Encode(&fix); err != nil {
23
+ if err := w.Encode(&fix); err != nil {
24
24
  return err
25
25
  }
26
26
  }
@@ -64,6 +64,6 @@ var _ = Describe("Producer", func() {
64
64
  info, err := obj.Head(ctx)
65
65
  Expect(err).NotTo(HaveOccurred())
66
66
  Expect(info.Size).To(BeNumerically("~", 75, 10))
67
- Expect(info.Metadata).To(HaveKeyWithValue("x-feedx-last-modified", "1515151515000"))
67
+ Expect(info.Metadata).To(HaveKeyWithValue("X-Feedx-Last-Modified", "1515151515000"))
68
68
  })
69
69
  })
data/reader.go CHANGED
@@ -2,7 +2,6 @@ package feedx
2
2
 
3
3
  import (
4
4
  "context"
5
- "fmt"
6
5
  "io"
7
6
  "time"
8
7
 
@@ -20,18 +19,13 @@ type ReaderOptions struct {
20
19
  Compression Compression
21
20
  }
22
21
 
23
- func (o *ReaderOptions) norm(name string) error {
22
+ func (o *ReaderOptions) norm(name string) {
24
23
  if o.Format == nil {
25
24
  o.Format = DetectFormat(name)
26
-
27
- if o.Format == nil {
28
- return fmt.Errorf("feedx: unable to detect format from %q", name)
29
- }
30
25
  }
31
26
  if o.Compression == nil {
32
27
  o.Compression = DetectCompression(name)
33
28
  }
34
- return nil
35
29
  }
36
30
 
37
31
  // Reader reads data from a remote feed.
@@ -52,9 +46,7 @@ func NewReader(ctx context.Context, remote *bfs.Object, opt *ReaderOptions) (*Re
52
46
  if opt != nil {
53
47
  o = *opt
54
48
  }
55
- if err := o.norm(remote.Name()); err != nil {
56
- return nil, err
57
- }
49
+ o.norm(remote.Name())
58
50
 
59
51
  return &Reader{
60
52
  remote: remote,
@@ -63,22 +55,19 @@ func NewReader(ctx context.Context, remote *bfs.Object, opt *ReaderOptions) (*Re
63
55
  }, nil
64
56
  }
65
57
 
66
- // Decode decodes the next value from the feed.
67
- func (r *Reader) Decode(v interface{}) error {
68
- if r.br == nil {
69
- br, err := r.remote.Open(r.ctx)
70
- if err != nil {
71
- return err
72
- }
73
- r.br = br
58
+ // Read reads raw bytes from the feed.
59
+ func (r *Reader) Read(p []byte) (int, error) {
60
+ if err := r.ensureOpen(); err != nil {
61
+ return 0, err
74
62
  }
75
63
 
76
- if r.cr == nil {
77
- cr, err := r.opt.Compression.NewReader(r.br)
78
- if err != nil {
79
- return err
80
- }
81
- r.cr = cr
64
+ return r.cr.Read(p)
65
+ }
66
+
67
+ // Decode decodes the next formatted value from the feed.
68
+ func (r *Reader) Decode(v interface{}) error {
69
+ if err := r.ensureOpen(); err != nil {
70
+ return err
82
71
  }
83
72
 
84
73
  if r.fd == nil {
@@ -128,3 +117,23 @@ func (r *Reader) Close() error {
128
117
  }
129
118
  return err
130
119
  }
120
+
121
+ func (r *Reader) ensureOpen() error {
122
+ if r.br == nil {
123
+ br, err := r.remote.Open(r.ctx)
124
+ if err != nil {
125
+ return err
126
+ }
127
+ r.br = br
128
+ }
129
+
130
+ if r.cr == nil {
131
+ cr, err := r.opt.Compression.NewReader(r.br)
132
+ if err != nil {
133
+ return err
134
+ }
135
+ r.cr = cr
136
+ }
137
+
138
+ return nil
139
+ }
@@ -3,6 +3,7 @@ package feedx_test
3
3
  import (
4
4
  "context"
5
5
  "io"
6
+ "io/ioutil"
6
7
 
7
8
  "github.com/bsm/feedx"
8
9
 
@@ -31,6 +32,13 @@ var _ = Describe("Reader", func() {
31
32
  })
32
33
 
33
34
  It("should read", func() {
35
+ data, err := ioutil.ReadAll(subject)
36
+ Expect(err).NotTo(HaveOccurred())
37
+ Expect(len(data)).To(BeNumerically("~", 140, 20))
38
+ Expect(subject.NumRead()).To(Equal(0))
39
+ })
40
+
41
+ It("should decode", func() {
34
42
  var msgs []tbp.Message
35
43
  for {
36
44
  var msg tbp.Message
@@ -48,7 +48,7 @@ RSpec.describe Feedx::Producer do
48
48
 
49
49
  it 'should support last-modified' do
50
50
  described_class.perform 'mock:///dir/file.json', last_modified: Time.at(1515151515), enum: enumerable
51
- expect(bucket.info('dir/file.json').metadata).to eq('x-feedx-last-modified' => '1515151515000')
51
+ expect(bucket.info('dir/file.json').metadata).to eq('X-Feedx-Last-Modified' => '1515151515000')
52
52
  end
53
53
 
54
54
  it 'should perform conditionally' do
@@ -39,7 +39,7 @@ RSpec.describe Feedx::Stream do
39
39
  subject.create metadata: { 'x' => '5' } do |s|
40
40
  s.encode(Feedx::TestCase::Model.new('X'))
41
41
  end
42
- expect(bucket.info('dir/file.json').metadata).to eq('x' => '5')
42
+ expect(bucket.info('dir/file.json').metadata).to eq('X' => '5')
43
43
  end
44
44
 
45
45
  it 'should decode' do
data/writer.go CHANGED
@@ -3,7 +3,6 @@ package feedx
3
3
  import (
4
4
  "bufio"
5
5
  "context"
6
- "fmt"
7
6
  "io"
8
7
  "time"
9
8
 
@@ -25,24 +24,16 @@ type WriterOptions struct {
25
24
  LastMod time.Time
26
25
  }
27
26
 
28
- func (o *WriterOptions) norm(name string) error {
27
+ func (o *WriterOptions) norm(name string) {
29
28
  if o.Format == nil {
30
29
  o.Format = DetectFormat(name)
31
-
32
- if o.Format == nil {
33
- return fmt.Errorf("feedx: unable to detect format from %q", name)
34
- }
35
30
  }
36
-
37
31
  if o.Compression == nil {
38
32
  o.Compression = DetectCompression(name)
39
33
  }
40
-
41
34
  if o.LastMod.IsZero() {
42
35
  o.LastMod = time.Now()
43
36
  }
44
-
45
- return nil
46
37
  }
47
38
 
48
39
  // Writer encodes feeds to remote locations.
@@ -77,29 +68,26 @@ func NewWriter(ctx context.Context, remote *bfs.Object, opt *WriterOptions) (*Wr
77
68
  }, nil
78
69
  }
79
70
 
80
- // Encode appends a value to the feed.
81
- func (w *Writer) Encode(v interface{}) error {
82
- if w.bw == nil {
83
- ts := timestampFromTime(w.opt.LastMod)
84
- bw, err := w.remote.Create(w.ctx, &bfs.WriteOptions{
85
- Metadata: map[string]string{metaLastModified: ts.String()},
86
- })
87
- if err != nil {
88
- return err
89
- }
90
- w.bw = bw
71
+ // Write write raw bytes to the feed.
72
+ func (w *Writer) Write(p []byte) (int, error) {
73
+ if err := w.ensureCreated(); err != nil {
74
+ return 0, err
91
75
  }
76
+ return w.ww.Write(p)
77
+ }
92
78
 
93
- if w.cw == nil {
94
- cw, err := w.opt.Compression.NewWriter(w.bw)
95
- if err != nil {
96
- return err
97
- }
98
- w.cw = cw
79
+ // WriteString write a raw string to the feed.
80
+ func (w *Writer) WriteString(s string) (int, error) {
81
+ if err := w.ensureCreated(); err != nil {
82
+ return 0, err
99
83
  }
84
+ return w.ww.WriteString(s)
85
+ }
100
86
 
101
- if w.ww == nil {
102
- w.ww = bufio.NewWriter(w.cw)
87
+ // Encode appends a value to the feed.
88
+ func (w *Writer) Encode(v interface{}) error {
89
+ if err := w.ensureCreated(); err != nil {
90
+ return err
103
91
  }
104
92
 
105
93
  if w.fe == nil {
@@ -154,3 +142,30 @@ func (w *Writer) Commit() error {
154
142
  }
155
143
  return err
156
144
  }
145
+
146
+ func (w *Writer) ensureCreated() error {
147
+ if w.bw == nil {
148
+ ts := timestampFromTime(w.opt.LastMod)
149
+ bw, err := w.remote.Create(w.ctx, &bfs.WriteOptions{
150
+ Metadata: bfs.Metadata{metaLastModified: ts.String()},
151
+ })
152
+ if err != nil {
153
+ return err
154
+ }
155
+ w.bw = bw
156
+ }
157
+
158
+ if w.cw == nil {
159
+ cw, err := w.opt.Compression.NewWriter(w.bw)
160
+ if err != nil {
161
+ return err
162
+ }
163
+ w.cw = cw
164
+ }
165
+
166
+ if w.ww == nil {
167
+ w.ww = bufio.NewWriter(w.cw)
168
+ }
169
+
170
+ return nil
171
+ }
@@ -1,9 +1,12 @@
1
1
  package feedx_test
2
2
 
3
3
  import (
4
+ "bytes"
4
5
  "context"
6
+ "time"
5
7
 
6
8
  "github.com/bsm/bfs"
9
+ "github.com/bsm/feedx"
7
10
  . "github.com/onsi/ginkgo"
8
11
  . "github.com/onsi/gomega"
9
12
  )
@@ -17,6 +20,38 @@ var _ = Describe("Writer", func() {
17
20
  compressed = bfs.NewInMemObject("path/to/file.jsonz")
18
21
  })
19
22
 
23
+ It("should write plain", func() {
24
+ w, err := feedx.NewWriter(context.Background(), plain, &feedx.WriterOptions{
25
+ LastMod: time.Unix(1515151515, 123456789),
26
+ })
27
+ Expect(err).NotTo(HaveOccurred())
28
+ defer w.Discard()
29
+
30
+ Expect(w.Write(bytes.Repeat([]byte{'x'}, 10000))).To(Equal(10000))
31
+ Expect(w.Commit()).To(Succeed())
32
+
33
+ info, err := plain.Head(ctx)
34
+ Expect(err).NotTo(HaveOccurred())
35
+ Expect(info.Size).To(Equal(int64(10000)))
36
+ Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
37
+ })
38
+
39
+ It("should write compressed", func() {
40
+ w, err := feedx.NewWriter(context.Background(), compressed, &feedx.WriterOptions{
41
+ LastMod: time.Unix(1515151515, 123456789),
42
+ })
43
+ Expect(err).NotTo(HaveOccurred())
44
+ defer w.Discard()
45
+
46
+ Expect(w.Write(bytes.Repeat([]byte{'x'}, 10000))).To(Equal(10000))
47
+ Expect(w.Commit()).To(Succeed())
48
+
49
+ info, err := compressed.Head(ctx)
50
+ Expect(err).NotTo(HaveOccurred())
51
+ Expect(info.Size).To(BeNumerically("~", 50, 20))
52
+ Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
53
+ })
54
+
20
55
  It("should encode", func() {
21
56
  Expect(writeMulti(plain, 10)).To(Succeed())
22
57
  Expect(writeMulti(compressed, 10)).To(Succeed())
@@ -24,11 +59,11 @@ var _ = Describe("Writer", func() {
24
59
  info, err := plain.Head(ctx)
25
60
  Expect(err).NotTo(HaveOccurred())
26
61
  Expect(info.Size).To(BeNumerically("~", 470, 10))
27
- Expect(info.Metadata).To(Equal(map[string]string{"x-feedx-last-modified": "1515151515123"}))
62
+ Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
28
63
 
29
64
  info, err = compressed.Head(ctx)
30
65
  Expect(err).NotTo(HaveOccurred())
31
66
  Expect(info.Size).To(BeNumerically("~", 76, 10))
32
- Expect(info.Metadata).To(Equal(map[string]string{"x-feedx-last-modified": "1515151515123"}))
67
+ Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
33
68
  })
34
69
  })
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: feedx
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.2
4
+ version: 0.8.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Black Square Media Ltd
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-06-25 00:00:00.000000000 Z
11
+ date: 2019-10-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bfs
@@ -16,14 +16,14 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 0.4.0
19
+ version: 0.5.0
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
- version: 0.4.0
26
+ version: 0.5.0
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: bundler
29
29
  requirement: !ruby/object:Gem::Requirement