feedx 0.7.2 → 0.8.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c294d0811dec65d1fce8a44fe2e18dbe08ac0bffee8d56c6b1d96bdac63e9366
4
- data.tar.gz: e6172cbf1f728569624a928dcf2a22c3a9a87824ed6cfd8fddaf834d1e526d8e
3
+ metadata.gz: 3d61930af3b031d191151820e5fefc106e3c64d8ece5d21513d8c91e4db195fb
4
+ data.tar.gz: 005bf01fefeb8b33299063f9c4e132ee9f1237f19a8bbf35c907e78863474121
5
5
  SHA512:
6
- metadata.gz: 64f69d35aa2cdca873e223b0f959a404dacebe03b07a8ca77796a10d5d6dfc06b4f7b9e7ee3bbfcfa03c23656399038324088a19d8e643636d0d3f3408af7b4b
7
- data.tar.gz: fb2e106e250693c5756edecf6d98df7115f8e8b33f4fca01ade0c3c568eb756215e6efa967d36099fdb055e4f77a26f68aebc207d9f62bb82c4748fb193d8ec7
6
+ metadata.gz: a4027d850b2122075bbb4d3ddc776fcd1be28f2f5db3d22c57a348d76fd821630554fe3327ac486b47b9b83e72c590476a68ab12eebc373d0fac231bb7ce449f
7
+ data.tar.gz: 857a3fc8cf13142a02634363e60478abd9171ef84c0d9a61f163c5debbea0a7a8378864d0e5ce27b4bdbe9a89d4b57df930c72a1da79602929a914a7ade38032
@@ -10,18 +10,13 @@ matrix:
10
10
  - 2.5
11
11
  before_install:
12
12
  - gem install bundler
13
- - language: ruby
14
- rvm:
15
- - 2.4
16
- before_install:
17
- - gem install bundler
18
13
  - language: go
19
14
  go:
20
- - 1.12.x
15
+ - 1.13.x
21
16
  env:
22
17
  - GO111MODULE=on
23
18
  - language: go
24
19
  go:
25
- - 1.11.x
20
+ - 1.12.x
26
21
  env:
27
22
  - GO111MODULE=on
@@ -1,45 +1,45 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- feedx (0.7.2)
5
- bfs (>= 0.4.0)
4
+ feedx (0.8.0)
5
+ bfs (>= 0.5.0)
6
6
 
7
7
  GEM
8
8
  remote: https://rubygems.org/
9
9
  specs:
10
10
  ast (2.4.0)
11
- bfs (0.4.0)
11
+ bfs (0.5.0)
12
12
  diff-lcs (1.3)
13
- google-protobuf (3.8.0)
13
+ google-protobuf (3.10.0-x86_64-linux)
14
14
  jaro_winkler (1.5.3)
15
- parallel (1.17.0)
16
- parser (2.6.3.0)
15
+ parallel (1.18.0)
16
+ parser (2.6.5.0)
17
17
  ast (~> 2.4.0)
18
18
  pbio (0.1.0)
19
19
  google-protobuf
20
20
  rainbow (3.0.0)
21
- rake (12.3.2)
22
- rspec (3.8.0)
23
- rspec-core (~> 3.8.0)
24
- rspec-expectations (~> 3.8.0)
25
- rspec-mocks (~> 3.8.0)
26
- rspec-core (3.8.1)
27
- rspec-support (~> 3.8.0)
28
- rspec-expectations (3.8.4)
21
+ rake (13.0.0)
22
+ rspec (3.9.0)
23
+ rspec-core (~> 3.9.0)
24
+ rspec-expectations (~> 3.9.0)
25
+ rspec-mocks (~> 3.9.0)
26
+ rspec-core (3.9.0)
27
+ rspec-support (~> 3.9.0)
28
+ rspec-expectations (3.9.0)
29
29
  diff-lcs (>= 1.2.0, < 2.0)
30
- rspec-support (~> 3.8.0)
31
- rspec-mocks (3.8.1)
30
+ rspec-support (~> 3.9.0)
31
+ rspec-mocks (3.9.0)
32
32
  diff-lcs (>= 1.2.0, < 2.0)
33
- rspec-support (~> 3.8.0)
34
- rspec-support (3.8.2)
35
- rubocop (0.71.0)
33
+ rspec-support (~> 3.9.0)
34
+ rspec-support (3.9.0)
35
+ rubocop (0.75.0)
36
36
  jaro_winkler (~> 1.5.1)
37
37
  parallel (~> 1.10)
38
38
  parser (>= 2.6)
39
39
  rainbow (>= 2.2.2, < 4.0)
40
40
  ruby-progressbar (~> 1.7)
41
41
  unicode-display_width (>= 1.4.0, < 1.7)
42
- rubocop-performance (1.4.0)
42
+ rubocop-performance (1.5.0)
43
43
  rubocop (>= 0.71.0)
44
44
  ruby-progressbar (1.10.1)
45
45
  unicode-display_width (1.6.0)
@@ -58,4 +58,4 @@ DEPENDENCIES
58
58
  rubocop-performance
59
59
 
60
60
  BUNDLED WITH
61
- 2.0.1
61
+ 2.0.2
data/README.md CHANGED
@@ -17,7 +17,7 @@ producer = Feedx::Producer.new relation, 's3://my-bucket/feeds/users.json.gz'
17
17
 
18
18
  # Push a new feed every hour
19
19
  loop do
20
- pusher.perform
20
+ producer.perform
21
21
  sleep(3600)
22
22
  end
23
23
  ```
@@ -21,12 +21,11 @@ type ConsumerOptions struct {
21
21
  AfterSync func(*ConsumerSync, error)
22
22
  }
23
23
 
24
- func (o *ConsumerOptions) norm(name string) error {
24
+ func (o *ConsumerOptions) norm(name string) {
25
25
  o.ReaderOptions.norm(name)
26
26
  if o.Interval <= 0 {
27
27
  o.Interval = time.Minute
28
28
  }
29
- return nil
30
29
  }
31
30
 
32
31
  // ConsumerSync contains the state of the last sync.
@@ -42,7 +41,7 @@ type ConsumerSync struct {
42
41
  }
43
42
 
44
43
  // ConsumeFunc is a parsing callback which is run by the consumer every sync interval.
45
- type ConsumeFunc func(FormatDecoder) (data interface{}, err error)
44
+ type ConsumeFunc func(*Reader) (data interface{}, err error)
46
45
 
47
46
  // Consumer manages data retrieval from a remote feed.
48
47
  // It queries the feed in regular intervals, continuously retrieving new updates.
@@ -81,9 +80,7 @@ func NewConsumerForRemote(ctx context.Context, remote *bfs.Object, opt *Consumer
81
80
  if opt != nil {
82
81
  o = *opt
83
82
  }
84
- if err := o.norm(remote.Name()); err != nil {
85
- return nil, err
86
- }
83
+ o.norm(remote.Name())
87
84
 
88
85
  ctx, stop := context.WithCancel(ctx)
89
86
  c := &consumer{
@@ -22,11 +22,11 @@ var _ = Describe("Consumer", func() {
22
22
  Expect(writeMulti(obj, 2)).To(Succeed())
23
23
 
24
24
  var err error
25
- subject, err = feedx.NewConsumerForRemote(ctx, obj, nil, func(dec feedx.FormatDecoder) (interface{}, error) {
25
+ subject, err = feedx.NewConsumerForRemote(ctx, obj, nil, func(r *feedx.Reader) (interface{}, error) {
26
26
  var msgs []tbp.Message
27
27
  for {
28
28
  var msg tbp.Message
29
- if err := dec.Decode(&msg); err == io.EOF {
29
+ if err := r.Decode(&msg); err == io.EOF {
30
30
  break
31
31
  }
32
32
  if err != nil {
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'feedx'
3
- s.version = '0.7.2'
3
+ s.version = '0.8.0'
4
4
  s.authors = ['Black Square Media Ltd']
5
5
  s.email = ['info@blacksquaremedia.com']
6
6
  s.summary = %(Exchange data between components via feeds)
@@ -13,7 +13,7 @@ Gem::Specification.new do |s|
13
13
  s.require_paths = ['lib']
14
14
  s.required_ruby_version = '>= 2.4'
15
15
 
16
- s.add_dependency 'bfs', '>= 0.4.0'
16
+ s.add_dependency 'bfs', '>= 0.5.0'
17
17
 
18
18
  s.add_development_dependency 'bundler'
19
19
  s.add_development_dependency 'pbio'
data/feedx.go CHANGED
@@ -13,8 +13,8 @@ import (
13
13
  var ErrNotModified = errors.New("feedx: not modified")
14
14
 
15
15
  const (
16
- metaLastModified = "x-feedx-last-modified"
17
- metaPusherLastModified = "x-feedx-pusher-last-modified"
16
+ metaLastModified = "X-Feedx-Last-Modified"
17
+ metaPusherLastModified = "X-Feedx-Pusher-Last-Modified"
18
18
  )
19
19
 
20
20
  // Timestamp with millisecond resolution
@@ -35,9 +35,9 @@ func remoteLastModified(ctx context.Context, obj *bfs.Object) (timestamp, error)
35
35
  return 0, err
36
36
  }
37
37
 
38
- millis, _ := strconv.ParseInt(info.Metadata[metaLastModified], 10, 64)
38
+ millis, _ := strconv.ParseInt(info.Metadata.Get(metaLastModified), 10, 64)
39
39
  if millis == 0 {
40
- millis, _ = strconv.ParseInt(info.Metadata[metaPusherLastModified], 10, 64)
40
+ millis, _ = strconv.ParseInt(info.Metadata.Get(metaPusherLastModified), 10, 64)
41
41
  }
42
42
  return timestamp(millis), nil
43
43
  }
data/format.go CHANGED
@@ -2,6 +2,7 @@ package feedx
2
2
 
3
3
  import (
4
4
  "encoding/json"
5
+ "errors"
5
6
  "fmt"
6
7
  "io"
7
8
  "path"
@@ -11,6 +12,8 @@ import (
11
12
  pbio "github.com/gogo/protobuf/io"
12
13
  )
13
14
 
15
+ var errNoFormat = errors.New("feedx: no format detected")
16
+
14
17
  // Format represents the data format.
15
18
  type Format interface {
16
19
  // NewDecoder wraps a decoder around a reader.
@@ -36,7 +39,7 @@ func DetectFormat(name string) Format {
36
39
  return DetectFormat(name[0 : len(name)-len(ext)])
37
40
  }
38
41
  }
39
- return nil
42
+ return (*noFormat)(nil)
40
43
  }
41
44
 
42
45
  // FormatDecoder methods
@@ -47,20 +50,23 @@ type FormatDecoder interface {
47
50
  io.Closer
48
51
  }
49
52
 
50
- // FormatPureEncoder methods
51
- type FormatPureEncoder interface {
53
+ // FormatEncoder methods
54
+ type FormatEncoder interface {
52
55
  // Encode encodes the value to the stream.
53
56
  Encode(v interface{}) error
54
- }
55
57
 
56
- // FormatEncoder methods
57
- type FormatEncoder interface {
58
- FormatPureEncoder
59
58
  io.Closer
60
59
  }
61
60
 
62
61
  // --------------------------------------------------------------------
63
62
 
63
+ type noFormat struct{}
64
+
65
+ func (*noFormat) NewDecoder(r io.Reader) (FormatDecoder, error) { return nil, errNoFormat }
66
+ func (*noFormat) NewEncoder(w io.Writer) (FormatEncoder, error) { return nil, errNoFormat }
67
+
68
+ // --------------------------------------------------------------------
69
+
64
70
  // JSONFormat provides a Format implemention for JSON.
65
71
  var JSONFormat = jsonFormat{}
66
72
 
data/go.mod CHANGED
@@ -1,9 +1,15 @@
1
1
  module github.com/bsm/feedx
2
2
 
3
3
  require (
4
- github.com/bsm/bfs v0.6.3
5
- github.com/gogo/protobuf v1.2.1
6
- github.com/golang/protobuf v1.3.1
7
- github.com/onsi/ginkgo v1.8.0
8
- github.com/onsi/gomega v1.5.0
4
+ github.com/bsm/bfs v0.8.0
5
+ github.com/gogo/protobuf v1.3.0
6
+ github.com/golang/protobuf v1.3.2
7
+ github.com/onsi/ginkgo v1.10.2
8
+ github.com/onsi/gomega v1.7.0
9
+ golang.org/x/net v0.0.0-20191007182048-72f939374954 // indirect
10
+ golang.org/x/sys v0.0.0-20191008105621-543471e840be // indirect
11
+ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
12
+ gopkg.in/yaml.v2 v2.2.4 // indirect
9
13
  )
14
+
15
+ go 1.13
data/go.sum CHANGED
@@ -1,18 +1,18 @@
1
- github.com/bmatcuk/doublestar v1.1.1 h1:YroD6BJCZBYx06yYFEWvUuKVWQn3vLLQAVmDmvTSaiQ=
2
- github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
3
- github.com/bsm/bfs v0.6.3 h1:qHOkJIx5W971wlnFLeJfiGGO0TUdB9JpTJ9RTAVntVg=
4
- github.com/bsm/bfs v0.6.3/go.mod h1:bRNSGIGvKb0MlboBpb7Aa3/Kaxmhes/6Be6wD+fsnKc=
1
+ github.com/bmatcuk/doublestar v1.1.5 h1:2bNwBOmhyFEFcoB3tGvTD5xanq+4kyOZlB8wFYbMjkk=
2
+ github.com/bmatcuk/doublestar v1.1.5/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
3
+ github.com/bsm/bfs v0.8.0 h1:/suMKytZ3WhVY62osdFm6VD+gJtxaohlUBuf76vGC78=
4
+ github.com/bsm/bfs v0.8.0/go.mod h1:cVv0jyqUY/jbHoG/WYPuWvOaOhW/HZ4jl7/JMlypvAE=
5
5
  github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
6
6
  github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
7
- github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
8
- github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
7
+ github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE=
8
+ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
9
9
  github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
10
10
  github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
11
- github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
12
- github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
11
+ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
12
+ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
13
13
  github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
14
14
  github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
15
- github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
15
+ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
16
16
  github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
17
17
  github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
18
18
  github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -23,26 +23,34 @@ github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
23
23
  github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
24
24
  github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
25
25
  github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
26
+ github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94=
27
+ github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
26
28
  github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
27
29
  github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
30
+ github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
31
+ github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
28
32
  golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
29
33
  golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
30
- golang.org/x/net v0.0.0-20190514140710-3ec191127204 h1:4yG6GqBtw9C+UrLp6s2wtSniayy/Vd/3F7ffLE427XI=
31
- golang.org/x/net v0.0.0-20190514140710-3ec191127204/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
34
+ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
35
+ golang.org/x/net v0.0.0-20191007182048-72f939374954 h1:JGZucVF/L/TotR719NbujzadOZ2AgnYlqphQGHDCKaU=
36
+ golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
32
37
  golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
33
38
  golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
34
39
  golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
35
- golang.org/x/sys v0.0.0-20190516110030-61b9204099cb h1:k07iPOt0d6nEnwXF+kHB+iEg+WSuKe/SOQuFM2QoD+E=
36
- golang.org/x/sys v0.0.0-20190516110030-61b9204099cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
40
+ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
41
+ golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU=
42
+ golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
37
43
  golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
38
44
  golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
39
45
  golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
40
46
  golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
41
- golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
42
47
  golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
48
+ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
43
49
  gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
44
50
  gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
45
51
  gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
52
+ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
53
+ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
46
54
  gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
47
55
  gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
48
56
  gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
@@ -51,3 +59,5 @@ gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
51
59
  gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
52
60
  gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
53
61
  gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
62
+ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
63
+ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -1,5 +1,6 @@
1
1
  module Feedx
2
- META_LAST_MODIFIED = 'x-feedx-last-modified'.freeze
2
+ META_LAST_MODIFIED = 'X-Feedx-Last-Modified'.freeze
3
+ META_LAST_MODIFIED_DC = META_LAST_MODIFIED.downcase.freeze
3
4
 
4
5
  autoload :Cache, 'feedx/cache'
5
6
  autoload :Compression, 'feedx/compression'
@@ -31,8 +31,9 @@ module Feedx
31
31
  remote_rev = nil
32
32
 
33
33
  if @cache
34
+ metadata = @stream.blob.info.metadata
34
35
  local_rev = @cache.read.to_i
35
- remote_rev = @stream.blob.info.metadata[META_LAST_MODIFIED].to_i
36
+ remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
36
37
  return false if remote_rev.positive? && remote_rev <= local_rev
37
38
  end
38
39
 
@@ -34,7 +34,8 @@ module Feedx
34
34
  local_rev = last_mod.is_a?(Integer) ? last_mod : (last_mod.to_f * 1000).floor
35
35
 
36
36
  begin
37
- remote_rev = @stream.blob.info.metadata[META_LAST_MODIFIED].to_i
37
+ metadata = @stream.blob.info.metadata
38
+ remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
38
39
  return -1 unless local_rev > remote_rev
39
40
  rescue BFS::FileNotFound # rubocop:disable Lint/HandleExceptions
40
41
  end if local_rev.positive?
@@ -9,7 +9,7 @@ import (
9
9
  )
10
10
 
11
11
  // ProduceFunc is a callback which is run by the producer on every iteration.
12
- type ProduceFunc func(FormatPureEncoder) error
12
+ type ProduceFunc func(*Writer) error
13
13
 
14
14
  // ProducerOptions configure the producer instance.
15
15
  type ProducerOptions struct {
@@ -28,12 +28,11 @@ type ProducerOptions struct {
28
28
  AfterPush func(*ProducerPush, error)
29
29
  }
30
30
 
31
- func (o *ProducerOptions) norm(name string) error {
31
+ func (o *ProducerOptions) norm(name string) {
32
32
  o.WriterOptions.norm(name)
33
33
  if o.Interval <= 0 {
34
34
  o.Interval = time.Minute
35
35
  }
36
- return nil
37
36
  }
38
37
 
39
38
  // ProducerPush contains the state of the last push.
@@ -79,9 +78,7 @@ func NewProducerForRemote(ctx context.Context, remote *bfs.Object, opt *Producer
79
78
  if opt != nil {
80
79
  o = *opt
81
80
  }
82
- if err := o.norm(remote.Name()); err != nil {
83
- return nil, err
84
- }
81
+ o.norm(remote.Name())
85
82
 
86
83
  ctx, stop := context.WithCancel(ctx)
87
84
  p := &Producer{
@@ -17,10 +17,10 @@ var _ = Describe("Producer", func() {
17
17
 
18
18
  setup := func(o *feedx.ProducerOptions) {
19
19
  var err error
20
- subject, err = feedx.NewProducerForRemote(ctx, obj, o, func(enc feedx.FormatPureEncoder) error {
20
+ subject, err = feedx.NewProducerForRemote(ctx, obj, o, func(w *feedx.Writer) error {
21
21
  for i := 0; i < 10; i++ {
22
22
  fix := fixture
23
- if err := enc.Encode(&fix); err != nil {
23
+ if err := w.Encode(&fix); err != nil {
24
24
  return err
25
25
  }
26
26
  }
@@ -64,6 +64,6 @@ var _ = Describe("Producer", func() {
64
64
  info, err := obj.Head(ctx)
65
65
  Expect(err).NotTo(HaveOccurred())
66
66
  Expect(info.Size).To(BeNumerically("~", 75, 10))
67
- Expect(info.Metadata).To(HaveKeyWithValue("x-feedx-last-modified", "1515151515000"))
67
+ Expect(info.Metadata).To(HaveKeyWithValue("X-Feedx-Last-Modified", "1515151515000"))
68
68
  })
69
69
  })
data/reader.go CHANGED
@@ -2,7 +2,6 @@ package feedx
2
2
 
3
3
  import (
4
4
  "context"
5
- "fmt"
6
5
  "io"
7
6
  "time"
8
7
 
@@ -20,18 +19,13 @@ type ReaderOptions struct {
20
19
  Compression Compression
21
20
  }
22
21
 
23
- func (o *ReaderOptions) norm(name string) error {
22
+ func (o *ReaderOptions) norm(name string) {
24
23
  if o.Format == nil {
25
24
  o.Format = DetectFormat(name)
26
-
27
- if o.Format == nil {
28
- return fmt.Errorf("feedx: unable to detect format from %q", name)
29
- }
30
25
  }
31
26
  if o.Compression == nil {
32
27
  o.Compression = DetectCompression(name)
33
28
  }
34
- return nil
35
29
  }
36
30
 
37
31
  // Reader reads data from a remote feed.
@@ -52,9 +46,7 @@ func NewReader(ctx context.Context, remote *bfs.Object, opt *ReaderOptions) (*Re
52
46
  if opt != nil {
53
47
  o = *opt
54
48
  }
55
- if err := o.norm(remote.Name()); err != nil {
56
- return nil, err
57
- }
49
+ o.norm(remote.Name())
58
50
 
59
51
  return &Reader{
60
52
  remote: remote,
@@ -63,22 +55,19 @@ func NewReader(ctx context.Context, remote *bfs.Object, opt *ReaderOptions) (*Re
63
55
  }, nil
64
56
  }
65
57
 
66
- // Decode decodes the next value from the feed.
67
- func (r *Reader) Decode(v interface{}) error {
68
- if r.br == nil {
69
- br, err := r.remote.Open(r.ctx)
70
- if err != nil {
71
- return err
72
- }
73
- r.br = br
58
+ // Read reads raw bytes from the feed.
59
+ func (r *Reader) Read(p []byte) (int, error) {
60
+ if err := r.ensureOpen(); err != nil {
61
+ return 0, err
74
62
  }
75
63
 
76
- if r.cr == nil {
77
- cr, err := r.opt.Compression.NewReader(r.br)
78
- if err != nil {
79
- return err
80
- }
81
- r.cr = cr
64
+ return r.cr.Read(p)
65
+ }
66
+
67
+ // Decode decodes the next formatted value from the feed.
68
+ func (r *Reader) Decode(v interface{}) error {
69
+ if err := r.ensureOpen(); err != nil {
70
+ return err
82
71
  }
83
72
 
84
73
  if r.fd == nil {
@@ -128,3 +117,23 @@ func (r *Reader) Close() error {
128
117
  }
129
118
  return err
130
119
  }
120
+
121
+ func (r *Reader) ensureOpen() error {
122
+ if r.br == nil {
123
+ br, err := r.remote.Open(r.ctx)
124
+ if err != nil {
125
+ return err
126
+ }
127
+ r.br = br
128
+ }
129
+
130
+ if r.cr == nil {
131
+ cr, err := r.opt.Compression.NewReader(r.br)
132
+ if err != nil {
133
+ return err
134
+ }
135
+ r.cr = cr
136
+ }
137
+
138
+ return nil
139
+ }
@@ -3,6 +3,7 @@ package feedx_test
3
3
  import (
4
4
  "context"
5
5
  "io"
6
+ "io/ioutil"
6
7
 
7
8
  "github.com/bsm/feedx"
8
9
 
@@ -31,6 +32,13 @@ var _ = Describe("Reader", func() {
31
32
  })
32
33
 
33
34
  It("should read", func() {
35
+ data, err := ioutil.ReadAll(subject)
36
+ Expect(err).NotTo(HaveOccurred())
37
+ Expect(len(data)).To(BeNumerically("~", 140, 20))
38
+ Expect(subject.NumRead()).To(Equal(0))
39
+ })
40
+
41
+ It("should decode", func() {
34
42
  var msgs []tbp.Message
35
43
  for {
36
44
  var msg tbp.Message
@@ -48,7 +48,7 @@ RSpec.describe Feedx::Producer do
48
48
 
49
49
  it 'should support last-modified' do
50
50
  described_class.perform 'mock:///dir/file.json', last_modified: Time.at(1515151515), enum: enumerable
51
- expect(bucket.info('dir/file.json').metadata).to eq('x-feedx-last-modified' => '1515151515000')
51
+ expect(bucket.info('dir/file.json').metadata).to eq('X-Feedx-Last-Modified' => '1515151515000')
52
52
  end
53
53
 
54
54
  it 'should perform conditionally' do
@@ -39,7 +39,7 @@ RSpec.describe Feedx::Stream do
39
39
  subject.create metadata: { 'x' => '5' } do |s|
40
40
  s.encode(Feedx::TestCase::Model.new('X'))
41
41
  end
42
- expect(bucket.info('dir/file.json').metadata).to eq('x' => '5')
42
+ expect(bucket.info('dir/file.json').metadata).to eq('X' => '5')
43
43
  end
44
44
 
45
45
  it 'should decode' do
data/writer.go CHANGED
@@ -3,7 +3,6 @@ package feedx
3
3
  import (
4
4
  "bufio"
5
5
  "context"
6
- "fmt"
7
6
  "io"
8
7
  "time"
9
8
 
@@ -25,24 +24,16 @@ type WriterOptions struct {
25
24
  LastMod time.Time
26
25
  }
27
26
 
28
- func (o *WriterOptions) norm(name string) error {
27
+ func (o *WriterOptions) norm(name string) {
29
28
  if o.Format == nil {
30
29
  o.Format = DetectFormat(name)
31
-
32
- if o.Format == nil {
33
- return fmt.Errorf("feedx: unable to detect format from %q", name)
34
- }
35
30
  }
36
-
37
31
  if o.Compression == nil {
38
32
  o.Compression = DetectCompression(name)
39
33
  }
40
-
41
34
  if o.LastMod.IsZero() {
42
35
  o.LastMod = time.Now()
43
36
  }
44
-
45
- return nil
46
37
  }
47
38
 
48
39
  // Writer encodes feeds to remote locations.
@@ -77,29 +68,26 @@ func NewWriter(ctx context.Context, remote *bfs.Object, opt *WriterOptions) (*Wr
77
68
  }, nil
78
69
  }
79
70
 
80
- // Encode appends a value to the feed.
81
- func (w *Writer) Encode(v interface{}) error {
82
- if w.bw == nil {
83
- ts := timestampFromTime(w.opt.LastMod)
84
- bw, err := w.remote.Create(w.ctx, &bfs.WriteOptions{
85
- Metadata: map[string]string{metaLastModified: ts.String()},
86
- })
87
- if err != nil {
88
- return err
89
- }
90
- w.bw = bw
71
+ // Write write raw bytes to the feed.
72
+ func (w *Writer) Write(p []byte) (int, error) {
73
+ if err := w.ensureCreated(); err != nil {
74
+ return 0, err
91
75
  }
76
+ return w.ww.Write(p)
77
+ }
92
78
 
93
- if w.cw == nil {
94
- cw, err := w.opt.Compression.NewWriter(w.bw)
95
- if err != nil {
96
- return err
97
- }
98
- w.cw = cw
79
+ // WriteString write a raw string to the feed.
80
+ func (w *Writer) WriteString(s string) (int, error) {
81
+ if err := w.ensureCreated(); err != nil {
82
+ return 0, err
99
83
  }
84
+ return w.ww.WriteString(s)
85
+ }
100
86
 
101
- if w.ww == nil {
102
- w.ww = bufio.NewWriter(w.cw)
87
+ // Encode appends a value to the feed.
88
+ func (w *Writer) Encode(v interface{}) error {
89
+ if err := w.ensureCreated(); err != nil {
90
+ return err
103
91
  }
104
92
 
105
93
  if w.fe == nil {
@@ -154,3 +142,30 @@ func (w *Writer) Commit() error {
154
142
  }
155
143
  return err
156
144
  }
145
+
146
+ func (w *Writer) ensureCreated() error {
147
+ if w.bw == nil {
148
+ ts := timestampFromTime(w.opt.LastMod)
149
+ bw, err := w.remote.Create(w.ctx, &bfs.WriteOptions{
150
+ Metadata: bfs.Metadata{metaLastModified: ts.String()},
151
+ })
152
+ if err != nil {
153
+ return err
154
+ }
155
+ w.bw = bw
156
+ }
157
+
158
+ if w.cw == nil {
159
+ cw, err := w.opt.Compression.NewWriter(w.bw)
160
+ if err != nil {
161
+ return err
162
+ }
163
+ w.cw = cw
164
+ }
165
+
166
+ if w.ww == nil {
167
+ w.ww = bufio.NewWriter(w.cw)
168
+ }
169
+
170
+ return nil
171
+ }
@@ -1,9 +1,12 @@
1
1
  package feedx_test
2
2
 
3
3
  import (
4
+ "bytes"
4
5
  "context"
6
+ "time"
5
7
 
6
8
  "github.com/bsm/bfs"
9
+ "github.com/bsm/feedx"
7
10
  . "github.com/onsi/ginkgo"
8
11
  . "github.com/onsi/gomega"
9
12
  )
@@ -17,6 +20,38 @@ var _ = Describe("Writer", func() {
17
20
  compressed = bfs.NewInMemObject("path/to/file.jsonz")
18
21
  })
19
22
 
23
+ It("should write plain", func() {
24
+ w, err := feedx.NewWriter(context.Background(), plain, &feedx.WriterOptions{
25
+ LastMod: time.Unix(1515151515, 123456789),
26
+ })
27
+ Expect(err).NotTo(HaveOccurred())
28
+ defer w.Discard()
29
+
30
+ Expect(w.Write(bytes.Repeat([]byte{'x'}, 10000))).To(Equal(10000))
31
+ Expect(w.Commit()).To(Succeed())
32
+
33
+ info, err := plain.Head(ctx)
34
+ Expect(err).NotTo(HaveOccurred())
35
+ Expect(info.Size).To(Equal(int64(10000)))
36
+ Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
37
+ })
38
+
39
+ It("should write compressed", func() {
40
+ w, err := feedx.NewWriter(context.Background(), compressed, &feedx.WriterOptions{
41
+ LastMod: time.Unix(1515151515, 123456789),
42
+ })
43
+ Expect(err).NotTo(HaveOccurred())
44
+ defer w.Discard()
45
+
46
+ Expect(w.Write(bytes.Repeat([]byte{'x'}, 10000))).To(Equal(10000))
47
+ Expect(w.Commit()).To(Succeed())
48
+
49
+ info, err := compressed.Head(ctx)
50
+ Expect(err).NotTo(HaveOccurred())
51
+ Expect(info.Size).To(BeNumerically("~", 50, 20))
52
+ Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
53
+ })
54
+
20
55
  It("should encode", func() {
21
56
  Expect(writeMulti(plain, 10)).To(Succeed())
22
57
  Expect(writeMulti(compressed, 10)).To(Succeed())
@@ -24,11 +59,11 @@ var _ = Describe("Writer", func() {
24
59
  info, err := plain.Head(ctx)
25
60
  Expect(err).NotTo(HaveOccurred())
26
61
  Expect(info.Size).To(BeNumerically("~", 470, 10))
27
- Expect(info.Metadata).To(Equal(map[string]string{"x-feedx-last-modified": "1515151515123"}))
62
+ Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
28
63
 
29
64
  info, err = compressed.Head(ctx)
30
65
  Expect(err).NotTo(HaveOccurred())
31
66
  Expect(info.Size).To(BeNumerically("~", 76, 10))
32
- Expect(info.Metadata).To(Equal(map[string]string{"x-feedx-last-modified": "1515151515123"}))
67
+ Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
33
68
  })
34
69
  })
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: feedx
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.2
4
+ version: 0.8.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Black Square Media Ltd
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-06-25 00:00:00.000000000 Z
11
+ date: 2019-10-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bfs
@@ -16,14 +16,14 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 0.4.0
19
+ version: 0.5.0
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
- version: 0.4.0
26
+ version: 0.5.0
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: bundler
29
29
  requirement: !ruby/object:Gem::Requirement