feedx 0.7.2 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +2 -7
- data/Gemfile.lock +21 -21
- data/README.md +1 -1
- data/consumer.go +3 -6
- data/consumer_test.go +2 -2
- data/feedx.gemspec +2 -2
- data/feedx.go +4 -4
- data/format.go +13 -7
- data/go.mod +11 -5
- data/go.sum +24 -14
- data/lib/feedx.rb +2 -1
- data/lib/feedx/consumer.rb +2 -1
- data/lib/feedx/producer.rb +2 -1
- data/producer.go +3 -6
- data/producer_test.go +3 -3
- data/reader.go +33 -24
- data/reader_test.go +8 -0
- data/spec/feedx/producer_spec.rb +1 -1
- data/spec/feedx/stream_spec.rb +1 -1
- data/writer.go +44 -29
- data/writer_test.go +37 -2
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3d61930af3b031d191151820e5fefc106e3c64d8ece5d21513d8c91e4db195fb
|
4
|
+
data.tar.gz: 005bf01fefeb8b33299063f9c4e132ee9f1237f19a8bbf35c907e78863474121
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a4027d850b2122075bbb4d3ddc776fcd1be28f2f5db3d22c57a348d76fd821630554fe3327ac486b47b9b83e72c590476a68ab12eebc373d0fac231bb7ce449f
|
7
|
+
data.tar.gz: 857a3fc8cf13142a02634363e60478abd9171ef84c0d9a61f163c5debbea0a7a8378864d0e5ce27b4bdbe9a89d4b57df930c72a1da79602929a914a7ade38032
|
data/.travis.yml
CHANGED
@@ -10,18 +10,13 @@ matrix:
|
|
10
10
|
- 2.5
|
11
11
|
before_install:
|
12
12
|
- gem install bundler
|
13
|
-
- language: ruby
|
14
|
-
rvm:
|
15
|
-
- 2.4
|
16
|
-
before_install:
|
17
|
-
- gem install bundler
|
18
13
|
- language: go
|
19
14
|
go:
|
20
|
-
- 1.
|
15
|
+
- 1.13.x
|
21
16
|
env:
|
22
17
|
- GO111MODULE=on
|
23
18
|
- language: go
|
24
19
|
go:
|
25
|
-
- 1.
|
20
|
+
- 1.12.x
|
26
21
|
env:
|
27
22
|
- GO111MODULE=on
|
data/Gemfile.lock
CHANGED
@@ -1,45 +1,45 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
feedx (0.
|
5
|
-
bfs (>= 0.
|
4
|
+
feedx (0.8.0)
|
5
|
+
bfs (>= 0.5.0)
|
6
6
|
|
7
7
|
GEM
|
8
8
|
remote: https://rubygems.org/
|
9
9
|
specs:
|
10
10
|
ast (2.4.0)
|
11
|
-
bfs (0.
|
11
|
+
bfs (0.5.0)
|
12
12
|
diff-lcs (1.3)
|
13
|
-
google-protobuf (3.
|
13
|
+
google-protobuf (3.10.0-x86_64-linux)
|
14
14
|
jaro_winkler (1.5.3)
|
15
|
-
parallel (1.
|
16
|
-
parser (2.6.
|
15
|
+
parallel (1.18.0)
|
16
|
+
parser (2.6.5.0)
|
17
17
|
ast (~> 2.4.0)
|
18
18
|
pbio (0.1.0)
|
19
19
|
google-protobuf
|
20
20
|
rainbow (3.0.0)
|
21
|
-
rake (
|
22
|
-
rspec (3.
|
23
|
-
rspec-core (~> 3.
|
24
|
-
rspec-expectations (~> 3.
|
25
|
-
rspec-mocks (~> 3.
|
26
|
-
rspec-core (3.
|
27
|
-
rspec-support (~> 3.
|
28
|
-
rspec-expectations (3.
|
21
|
+
rake (13.0.0)
|
22
|
+
rspec (3.9.0)
|
23
|
+
rspec-core (~> 3.9.0)
|
24
|
+
rspec-expectations (~> 3.9.0)
|
25
|
+
rspec-mocks (~> 3.9.0)
|
26
|
+
rspec-core (3.9.0)
|
27
|
+
rspec-support (~> 3.9.0)
|
28
|
+
rspec-expectations (3.9.0)
|
29
29
|
diff-lcs (>= 1.2.0, < 2.0)
|
30
|
-
rspec-support (~> 3.
|
31
|
-
rspec-mocks (3.
|
30
|
+
rspec-support (~> 3.9.0)
|
31
|
+
rspec-mocks (3.9.0)
|
32
32
|
diff-lcs (>= 1.2.0, < 2.0)
|
33
|
-
rspec-support (~> 3.
|
34
|
-
rspec-support (3.
|
35
|
-
rubocop (0.
|
33
|
+
rspec-support (~> 3.9.0)
|
34
|
+
rspec-support (3.9.0)
|
35
|
+
rubocop (0.75.0)
|
36
36
|
jaro_winkler (~> 1.5.1)
|
37
37
|
parallel (~> 1.10)
|
38
38
|
parser (>= 2.6)
|
39
39
|
rainbow (>= 2.2.2, < 4.0)
|
40
40
|
ruby-progressbar (~> 1.7)
|
41
41
|
unicode-display_width (>= 1.4.0, < 1.7)
|
42
|
-
rubocop-performance (1.
|
42
|
+
rubocop-performance (1.5.0)
|
43
43
|
rubocop (>= 0.71.0)
|
44
44
|
ruby-progressbar (1.10.1)
|
45
45
|
unicode-display_width (1.6.0)
|
@@ -58,4 +58,4 @@ DEPENDENCIES
|
|
58
58
|
rubocop-performance
|
59
59
|
|
60
60
|
BUNDLED WITH
|
61
|
-
2.0.
|
61
|
+
2.0.2
|
data/README.md
CHANGED
data/consumer.go
CHANGED
@@ -21,12 +21,11 @@ type ConsumerOptions struct {
|
|
21
21
|
AfterSync func(*ConsumerSync, error)
|
22
22
|
}
|
23
23
|
|
24
|
-
func (o *ConsumerOptions) norm(name string)
|
24
|
+
func (o *ConsumerOptions) norm(name string) {
|
25
25
|
o.ReaderOptions.norm(name)
|
26
26
|
if o.Interval <= 0 {
|
27
27
|
o.Interval = time.Minute
|
28
28
|
}
|
29
|
-
return nil
|
30
29
|
}
|
31
30
|
|
32
31
|
// ConsumerSync contains the state of the last sync.
|
@@ -42,7 +41,7 @@ type ConsumerSync struct {
|
|
42
41
|
}
|
43
42
|
|
44
43
|
// ConsumeFunc is a parsing callback which is run by the consumer every sync interval.
|
45
|
-
type ConsumeFunc func(
|
44
|
+
type ConsumeFunc func(*Reader) (data interface{}, err error)
|
46
45
|
|
47
46
|
// Consumer manages data retrieval from a remote feed.
|
48
47
|
// It queries the feed in regular intervals, continuously retrieving new updates.
|
@@ -81,9 +80,7 @@ func NewConsumerForRemote(ctx context.Context, remote *bfs.Object, opt *Consumer
|
|
81
80
|
if opt != nil {
|
82
81
|
o = *opt
|
83
82
|
}
|
84
|
-
|
85
|
-
return nil, err
|
86
|
-
}
|
83
|
+
o.norm(remote.Name())
|
87
84
|
|
88
85
|
ctx, stop := context.WithCancel(ctx)
|
89
86
|
c := &consumer{
|
data/consumer_test.go
CHANGED
@@ -22,11 +22,11 @@ var _ = Describe("Consumer", func() {
|
|
22
22
|
Expect(writeMulti(obj, 2)).To(Succeed())
|
23
23
|
|
24
24
|
var err error
|
25
|
-
subject, err = feedx.NewConsumerForRemote(ctx, obj, nil, func(
|
25
|
+
subject, err = feedx.NewConsumerForRemote(ctx, obj, nil, func(r *feedx.Reader) (interface{}, error) {
|
26
26
|
var msgs []tbp.Message
|
27
27
|
for {
|
28
28
|
var msg tbp.Message
|
29
|
-
if err :=
|
29
|
+
if err := r.Decode(&msg); err == io.EOF {
|
30
30
|
break
|
31
31
|
}
|
32
32
|
if err != nil {
|
data/feedx.gemspec
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'feedx'
|
3
|
-
s.version = '0.
|
3
|
+
s.version = '0.8.0'
|
4
4
|
s.authors = ['Black Square Media Ltd']
|
5
5
|
s.email = ['info@blacksquaremedia.com']
|
6
6
|
s.summary = %(Exchange data between components via feeds)
|
@@ -13,7 +13,7 @@ Gem::Specification.new do |s|
|
|
13
13
|
s.require_paths = ['lib']
|
14
14
|
s.required_ruby_version = '>= 2.4'
|
15
15
|
|
16
|
-
s.add_dependency 'bfs', '>= 0.
|
16
|
+
s.add_dependency 'bfs', '>= 0.5.0'
|
17
17
|
|
18
18
|
s.add_development_dependency 'bundler'
|
19
19
|
s.add_development_dependency 'pbio'
|
data/feedx.go
CHANGED
@@ -13,8 +13,8 @@ import (
|
|
13
13
|
var ErrNotModified = errors.New("feedx: not modified")
|
14
14
|
|
15
15
|
const (
|
16
|
-
metaLastModified = "
|
17
|
-
metaPusherLastModified = "
|
16
|
+
metaLastModified = "X-Feedx-Last-Modified"
|
17
|
+
metaPusherLastModified = "X-Feedx-Pusher-Last-Modified"
|
18
18
|
)
|
19
19
|
|
20
20
|
// Timestamp with millisecond resolution
|
@@ -35,9 +35,9 @@ func remoteLastModified(ctx context.Context, obj *bfs.Object) (timestamp, error)
|
|
35
35
|
return 0, err
|
36
36
|
}
|
37
37
|
|
38
|
-
millis, _ := strconv.ParseInt(info.Metadata
|
38
|
+
millis, _ := strconv.ParseInt(info.Metadata.Get(metaLastModified), 10, 64)
|
39
39
|
if millis == 0 {
|
40
|
-
millis, _ = strconv.ParseInt(info.Metadata
|
40
|
+
millis, _ = strconv.ParseInt(info.Metadata.Get(metaPusherLastModified), 10, 64)
|
41
41
|
}
|
42
42
|
return timestamp(millis), nil
|
43
43
|
}
|
data/format.go
CHANGED
@@ -2,6 +2,7 @@ package feedx
|
|
2
2
|
|
3
3
|
import (
|
4
4
|
"encoding/json"
|
5
|
+
"errors"
|
5
6
|
"fmt"
|
6
7
|
"io"
|
7
8
|
"path"
|
@@ -11,6 +12,8 @@ import (
|
|
11
12
|
pbio "github.com/gogo/protobuf/io"
|
12
13
|
)
|
13
14
|
|
15
|
+
var errNoFormat = errors.New("feedx: no format detected")
|
16
|
+
|
14
17
|
// Format represents the data format.
|
15
18
|
type Format interface {
|
16
19
|
// NewDecoder wraps a decoder around a reader.
|
@@ -36,7 +39,7 @@ func DetectFormat(name string) Format {
|
|
36
39
|
return DetectFormat(name[0 : len(name)-len(ext)])
|
37
40
|
}
|
38
41
|
}
|
39
|
-
return nil
|
42
|
+
return (*noFormat)(nil)
|
40
43
|
}
|
41
44
|
|
42
45
|
// FormatDecoder methods
|
@@ -47,20 +50,23 @@ type FormatDecoder interface {
|
|
47
50
|
io.Closer
|
48
51
|
}
|
49
52
|
|
50
|
-
//
|
51
|
-
type
|
53
|
+
// FormatEncoder methods
|
54
|
+
type FormatEncoder interface {
|
52
55
|
// Encode encodes the value to the stream.
|
53
56
|
Encode(v interface{}) error
|
54
|
-
}
|
55
57
|
|
56
|
-
// FormatEncoder methods
|
57
|
-
type FormatEncoder interface {
|
58
|
-
FormatPureEncoder
|
59
58
|
io.Closer
|
60
59
|
}
|
61
60
|
|
62
61
|
// --------------------------------------------------------------------
|
63
62
|
|
63
|
+
type noFormat struct{}
|
64
|
+
|
65
|
+
func (*noFormat) NewDecoder(r io.Reader) (FormatDecoder, error) { return nil, errNoFormat }
|
66
|
+
func (*noFormat) NewEncoder(w io.Writer) (FormatEncoder, error) { return nil, errNoFormat }
|
67
|
+
|
68
|
+
// --------------------------------------------------------------------
|
69
|
+
|
64
70
|
// JSONFormat provides a Format implemention for JSON.
|
65
71
|
var JSONFormat = jsonFormat{}
|
66
72
|
|
data/go.mod
CHANGED
@@ -1,9 +1,15 @@
|
|
1
1
|
module github.com/bsm/feedx
|
2
2
|
|
3
3
|
require (
|
4
|
-
github.com/bsm/bfs v0.
|
5
|
-
github.com/gogo/protobuf v1.
|
6
|
-
github.com/golang/protobuf v1.3.
|
7
|
-
github.com/onsi/ginkgo v1.
|
8
|
-
github.com/onsi/gomega v1.
|
4
|
+
github.com/bsm/bfs v0.8.0
|
5
|
+
github.com/gogo/protobuf v1.3.0
|
6
|
+
github.com/golang/protobuf v1.3.2
|
7
|
+
github.com/onsi/ginkgo v1.10.2
|
8
|
+
github.com/onsi/gomega v1.7.0
|
9
|
+
golang.org/x/net v0.0.0-20191007182048-72f939374954 // indirect
|
10
|
+
golang.org/x/sys v0.0.0-20191008105621-543471e840be // indirect
|
11
|
+
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
|
12
|
+
gopkg.in/yaml.v2 v2.2.4 // indirect
|
9
13
|
)
|
14
|
+
|
15
|
+
go 1.13
|
data/go.sum
CHANGED
@@ -1,18 +1,18 @@
|
|
1
|
-
github.com/bmatcuk/doublestar v1.1.
|
2
|
-
github.com/bmatcuk/doublestar v1.1.
|
3
|
-
github.com/bsm/bfs v0.
|
4
|
-
github.com/bsm/bfs v0.
|
1
|
+
github.com/bmatcuk/doublestar v1.1.5 h1:2bNwBOmhyFEFcoB3tGvTD5xanq+4kyOZlB8wFYbMjkk=
|
2
|
+
github.com/bmatcuk/doublestar v1.1.5/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
|
3
|
+
github.com/bsm/bfs v0.8.0 h1:/suMKytZ3WhVY62osdFm6VD+gJtxaohlUBuf76vGC78=
|
4
|
+
github.com/bsm/bfs v0.8.0/go.mod h1:cVv0jyqUY/jbHoG/WYPuWvOaOhW/HZ4jl7/JMlypvAE=
|
5
5
|
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
6
6
|
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
7
|
-
github.com/gogo/protobuf v1.
|
8
|
-
github.com/gogo/protobuf v1.
|
7
|
+
github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE=
|
8
|
+
github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
|
9
9
|
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
|
10
10
|
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
11
|
-
github.com/golang/protobuf v1.3.
|
12
|
-
github.com/golang/protobuf v1.3.
|
11
|
+
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
|
12
|
+
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
13
13
|
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
14
14
|
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
15
|
-
github.com/kisielk/errcheck v1.
|
15
|
+
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
|
16
16
|
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
17
17
|
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
18
18
|
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
@@ -23,26 +23,34 @@ github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
|
|
23
23
|
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
24
24
|
github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
|
25
25
|
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
26
|
+
github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94=
|
27
|
+
github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
26
28
|
github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
|
27
29
|
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
30
|
+
github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
|
31
|
+
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
28
32
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
29
33
|
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
30
|
-
golang.org/x/net v0.0.0-
|
31
|
-
golang.org/x/net v0.0.0-
|
34
|
+
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
35
|
+
golang.org/x/net v0.0.0-20191007182048-72f939374954 h1:JGZucVF/L/TotR719NbujzadOZ2AgnYlqphQGHDCKaU=
|
36
|
+
golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
32
37
|
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
33
38
|
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
34
39
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
35
|
-
golang.org/x/sys v0.0.0-
|
36
|
-
golang.org/x/sys v0.0.0-
|
40
|
+
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
41
|
+
golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU=
|
42
|
+
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
37
43
|
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
|
38
44
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
39
45
|
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
40
46
|
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
41
|
-
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
42
47
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
48
|
+
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
43
49
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
44
50
|
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
45
51
|
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
52
|
+
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
53
|
+
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
46
54
|
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
47
55
|
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
48
56
|
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
@@ -51,3 +59,5 @@ gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
|
|
51
59
|
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
52
60
|
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
53
61
|
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
62
|
+
gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
|
63
|
+
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
data/lib/feedx.rb
CHANGED
data/lib/feedx/consumer.rb
CHANGED
@@ -31,8 +31,9 @@ module Feedx
|
|
31
31
|
remote_rev = nil
|
32
32
|
|
33
33
|
if @cache
|
34
|
+
metadata = @stream.blob.info.metadata
|
34
35
|
local_rev = @cache.read.to_i
|
35
|
-
remote_rev =
|
36
|
+
remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
|
36
37
|
return false if remote_rev.positive? && remote_rev <= local_rev
|
37
38
|
end
|
38
39
|
|
data/lib/feedx/producer.rb
CHANGED
@@ -34,7 +34,8 @@ module Feedx
|
|
34
34
|
local_rev = last_mod.is_a?(Integer) ? last_mod : (last_mod.to_f * 1000).floor
|
35
35
|
|
36
36
|
begin
|
37
|
-
|
37
|
+
metadata = @stream.blob.info.metadata
|
38
|
+
remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
|
38
39
|
return -1 unless local_rev > remote_rev
|
39
40
|
rescue BFS::FileNotFound # rubocop:disable Lint/HandleExceptions
|
40
41
|
end if local_rev.positive?
|
data/producer.go
CHANGED
@@ -9,7 +9,7 @@ import (
|
|
9
9
|
)
|
10
10
|
|
11
11
|
// ProduceFunc is a callback which is run by the producer on every iteration.
|
12
|
-
type ProduceFunc func(
|
12
|
+
type ProduceFunc func(*Writer) error
|
13
13
|
|
14
14
|
// ProducerOptions configure the producer instance.
|
15
15
|
type ProducerOptions struct {
|
@@ -28,12 +28,11 @@ type ProducerOptions struct {
|
|
28
28
|
AfterPush func(*ProducerPush, error)
|
29
29
|
}
|
30
30
|
|
31
|
-
func (o *ProducerOptions) norm(name string)
|
31
|
+
func (o *ProducerOptions) norm(name string) {
|
32
32
|
o.WriterOptions.norm(name)
|
33
33
|
if o.Interval <= 0 {
|
34
34
|
o.Interval = time.Minute
|
35
35
|
}
|
36
|
-
return nil
|
37
36
|
}
|
38
37
|
|
39
38
|
// ProducerPush contains the state of the last push.
|
@@ -79,9 +78,7 @@ func NewProducerForRemote(ctx context.Context, remote *bfs.Object, opt *Producer
|
|
79
78
|
if opt != nil {
|
80
79
|
o = *opt
|
81
80
|
}
|
82
|
-
|
83
|
-
return nil, err
|
84
|
-
}
|
81
|
+
o.norm(remote.Name())
|
85
82
|
|
86
83
|
ctx, stop := context.WithCancel(ctx)
|
87
84
|
p := &Producer{
|
data/producer_test.go
CHANGED
@@ -17,10 +17,10 @@ var _ = Describe("Producer", func() {
|
|
17
17
|
|
18
18
|
setup := func(o *feedx.ProducerOptions) {
|
19
19
|
var err error
|
20
|
-
subject, err = feedx.NewProducerForRemote(ctx, obj, o, func(
|
20
|
+
subject, err = feedx.NewProducerForRemote(ctx, obj, o, func(w *feedx.Writer) error {
|
21
21
|
for i := 0; i < 10; i++ {
|
22
22
|
fix := fixture
|
23
|
-
if err :=
|
23
|
+
if err := w.Encode(&fix); err != nil {
|
24
24
|
return err
|
25
25
|
}
|
26
26
|
}
|
@@ -64,6 +64,6 @@ var _ = Describe("Producer", func() {
|
|
64
64
|
info, err := obj.Head(ctx)
|
65
65
|
Expect(err).NotTo(HaveOccurred())
|
66
66
|
Expect(info.Size).To(BeNumerically("~", 75, 10))
|
67
|
-
Expect(info.Metadata).To(HaveKeyWithValue("
|
67
|
+
Expect(info.Metadata).To(HaveKeyWithValue("X-Feedx-Last-Modified", "1515151515000"))
|
68
68
|
})
|
69
69
|
})
|
data/reader.go
CHANGED
@@ -2,7 +2,6 @@ package feedx
|
|
2
2
|
|
3
3
|
import (
|
4
4
|
"context"
|
5
|
-
"fmt"
|
6
5
|
"io"
|
7
6
|
"time"
|
8
7
|
|
@@ -20,18 +19,13 @@ type ReaderOptions struct {
|
|
20
19
|
Compression Compression
|
21
20
|
}
|
22
21
|
|
23
|
-
func (o *ReaderOptions) norm(name string)
|
22
|
+
func (o *ReaderOptions) norm(name string) {
|
24
23
|
if o.Format == nil {
|
25
24
|
o.Format = DetectFormat(name)
|
26
|
-
|
27
|
-
if o.Format == nil {
|
28
|
-
return fmt.Errorf("feedx: unable to detect format from %q", name)
|
29
|
-
}
|
30
25
|
}
|
31
26
|
if o.Compression == nil {
|
32
27
|
o.Compression = DetectCompression(name)
|
33
28
|
}
|
34
|
-
return nil
|
35
29
|
}
|
36
30
|
|
37
31
|
// Reader reads data from a remote feed.
|
@@ -52,9 +46,7 @@ func NewReader(ctx context.Context, remote *bfs.Object, opt *ReaderOptions) (*Re
|
|
52
46
|
if opt != nil {
|
53
47
|
o = *opt
|
54
48
|
}
|
55
|
-
|
56
|
-
return nil, err
|
57
|
-
}
|
49
|
+
o.norm(remote.Name())
|
58
50
|
|
59
51
|
return &Reader{
|
60
52
|
remote: remote,
|
@@ -63,22 +55,19 @@ func NewReader(ctx context.Context, remote *bfs.Object, opt *ReaderOptions) (*Re
|
|
63
55
|
}, nil
|
64
56
|
}
|
65
57
|
|
66
|
-
//
|
67
|
-
func (r *Reader)
|
68
|
-
if r.
|
69
|
-
|
70
|
-
if err != nil {
|
71
|
-
return err
|
72
|
-
}
|
73
|
-
r.br = br
|
58
|
+
// Read reads raw bytes from the feed.
|
59
|
+
func (r *Reader) Read(p []byte) (int, error) {
|
60
|
+
if err := r.ensureOpen(); err != nil {
|
61
|
+
return 0, err
|
74
62
|
}
|
75
63
|
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
64
|
+
return r.cr.Read(p)
|
65
|
+
}
|
66
|
+
|
67
|
+
// Decode decodes the next formatted value from the feed.
|
68
|
+
func (r *Reader) Decode(v interface{}) error {
|
69
|
+
if err := r.ensureOpen(); err != nil {
|
70
|
+
return err
|
82
71
|
}
|
83
72
|
|
84
73
|
if r.fd == nil {
|
@@ -128,3 +117,23 @@ func (r *Reader) Close() error {
|
|
128
117
|
}
|
129
118
|
return err
|
130
119
|
}
|
120
|
+
|
121
|
+
func (r *Reader) ensureOpen() error {
|
122
|
+
if r.br == nil {
|
123
|
+
br, err := r.remote.Open(r.ctx)
|
124
|
+
if err != nil {
|
125
|
+
return err
|
126
|
+
}
|
127
|
+
r.br = br
|
128
|
+
}
|
129
|
+
|
130
|
+
if r.cr == nil {
|
131
|
+
cr, err := r.opt.Compression.NewReader(r.br)
|
132
|
+
if err != nil {
|
133
|
+
return err
|
134
|
+
}
|
135
|
+
r.cr = cr
|
136
|
+
}
|
137
|
+
|
138
|
+
return nil
|
139
|
+
}
|
data/reader_test.go
CHANGED
@@ -3,6 +3,7 @@ package feedx_test
|
|
3
3
|
import (
|
4
4
|
"context"
|
5
5
|
"io"
|
6
|
+
"io/ioutil"
|
6
7
|
|
7
8
|
"github.com/bsm/feedx"
|
8
9
|
|
@@ -31,6 +32,13 @@ var _ = Describe("Reader", func() {
|
|
31
32
|
})
|
32
33
|
|
33
34
|
It("should read", func() {
|
35
|
+
data, err := ioutil.ReadAll(subject)
|
36
|
+
Expect(err).NotTo(HaveOccurred())
|
37
|
+
Expect(len(data)).To(BeNumerically("~", 140, 20))
|
38
|
+
Expect(subject.NumRead()).To(Equal(0))
|
39
|
+
})
|
40
|
+
|
41
|
+
It("should decode", func() {
|
34
42
|
var msgs []tbp.Message
|
35
43
|
for {
|
36
44
|
var msg tbp.Message
|
data/spec/feedx/producer_spec.rb
CHANGED
@@ -48,7 +48,7 @@ RSpec.describe Feedx::Producer do
|
|
48
48
|
|
49
49
|
it 'should support last-modified' do
|
50
50
|
described_class.perform 'mock:///dir/file.json', last_modified: Time.at(1515151515), enum: enumerable
|
51
|
-
expect(bucket.info('dir/file.json').metadata).to eq('
|
51
|
+
expect(bucket.info('dir/file.json').metadata).to eq('X-Feedx-Last-Modified' => '1515151515000')
|
52
52
|
end
|
53
53
|
|
54
54
|
it 'should perform conditionally' do
|
data/spec/feedx/stream_spec.rb
CHANGED
@@ -39,7 +39,7 @@ RSpec.describe Feedx::Stream do
|
|
39
39
|
subject.create metadata: { 'x' => '5' } do |s|
|
40
40
|
s.encode(Feedx::TestCase::Model.new('X'))
|
41
41
|
end
|
42
|
-
expect(bucket.info('dir/file.json').metadata).to eq('
|
42
|
+
expect(bucket.info('dir/file.json').metadata).to eq('X' => '5')
|
43
43
|
end
|
44
44
|
|
45
45
|
it 'should decode' do
|
data/writer.go
CHANGED
@@ -3,7 +3,6 @@ package feedx
|
|
3
3
|
import (
|
4
4
|
"bufio"
|
5
5
|
"context"
|
6
|
-
"fmt"
|
7
6
|
"io"
|
8
7
|
"time"
|
9
8
|
|
@@ -25,24 +24,16 @@ type WriterOptions struct {
|
|
25
24
|
LastMod time.Time
|
26
25
|
}
|
27
26
|
|
28
|
-
func (o *WriterOptions) norm(name string)
|
27
|
+
func (o *WriterOptions) norm(name string) {
|
29
28
|
if o.Format == nil {
|
30
29
|
o.Format = DetectFormat(name)
|
31
|
-
|
32
|
-
if o.Format == nil {
|
33
|
-
return fmt.Errorf("feedx: unable to detect format from %q", name)
|
34
|
-
}
|
35
30
|
}
|
36
|
-
|
37
31
|
if o.Compression == nil {
|
38
32
|
o.Compression = DetectCompression(name)
|
39
33
|
}
|
40
|
-
|
41
34
|
if o.LastMod.IsZero() {
|
42
35
|
o.LastMod = time.Now()
|
43
36
|
}
|
44
|
-
|
45
|
-
return nil
|
46
37
|
}
|
47
38
|
|
48
39
|
// Writer encodes feeds to remote locations.
|
@@ -77,29 +68,26 @@ func NewWriter(ctx context.Context, remote *bfs.Object, opt *WriterOptions) (*Wr
|
|
77
68
|
}, nil
|
78
69
|
}
|
79
70
|
|
80
|
-
//
|
81
|
-
func (w *Writer)
|
82
|
-
if w.
|
83
|
-
|
84
|
-
bw, err := w.remote.Create(w.ctx, &bfs.WriteOptions{
|
85
|
-
Metadata: map[string]string{metaLastModified: ts.String()},
|
86
|
-
})
|
87
|
-
if err != nil {
|
88
|
-
return err
|
89
|
-
}
|
90
|
-
w.bw = bw
|
71
|
+
// Write write raw bytes to the feed.
|
72
|
+
func (w *Writer) Write(p []byte) (int, error) {
|
73
|
+
if err := w.ensureCreated(); err != nil {
|
74
|
+
return 0, err
|
91
75
|
}
|
76
|
+
return w.ww.Write(p)
|
77
|
+
}
|
92
78
|
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
}
|
98
|
-
w.cw = cw
|
79
|
+
// WriteString write a raw string to the feed.
|
80
|
+
func (w *Writer) WriteString(s string) (int, error) {
|
81
|
+
if err := w.ensureCreated(); err != nil {
|
82
|
+
return 0, err
|
99
83
|
}
|
84
|
+
return w.ww.WriteString(s)
|
85
|
+
}
|
100
86
|
|
101
|
-
|
102
|
-
|
87
|
+
// Encode appends a value to the feed.
|
88
|
+
func (w *Writer) Encode(v interface{}) error {
|
89
|
+
if err := w.ensureCreated(); err != nil {
|
90
|
+
return err
|
103
91
|
}
|
104
92
|
|
105
93
|
if w.fe == nil {
|
@@ -154,3 +142,30 @@ func (w *Writer) Commit() error {
|
|
154
142
|
}
|
155
143
|
return err
|
156
144
|
}
|
145
|
+
|
146
|
+
func (w *Writer) ensureCreated() error {
|
147
|
+
if w.bw == nil {
|
148
|
+
ts := timestampFromTime(w.opt.LastMod)
|
149
|
+
bw, err := w.remote.Create(w.ctx, &bfs.WriteOptions{
|
150
|
+
Metadata: bfs.Metadata{metaLastModified: ts.String()},
|
151
|
+
})
|
152
|
+
if err != nil {
|
153
|
+
return err
|
154
|
+
}
|
155
|
+
w.bw = bw
|
156
|
+
}
|
157
|
+
|
158
|
+
if w.cw == nil {
|
159
|
+
cw, err := w.opt.Compression.NewWriter(w.bw)
|
160
|
+
if err != nil {
|
161
|
+
return err
|
162
|
+
}
|
163
|
+
w.cw = cw
|
164
|
+
}
|
165
|
+
|
166
|
+
if w.ww == nil {
|
167
|
+
w.ww = bufio.NewWriter(w.cw)
|
168
|
+
}
|
169
|
+
|
170
|
+
return nil
|
171
|
+
}
|
data/writer_test.go
CHANGED
@@ -1,9 +1,12 @@
|
|
1
1
|
package feedx_test
|
2
2
|
|
3
3
|
import (
|
4
|
+
"bytes"
|
4
5
|
"context"
|
6
|
+
"time"
|
5
7
|
|
6
8
|
"github.com/bsm/bfs"
|
9
|
+
"github.com/bsm/feedx"
|
7
10
|
. "github.com/onsi/ginkgo"
|
8
11
|
. "github.com/onsi/gomega"
|
9
12
|
)
|
@@ -17,6 +20,38 @@ var _ = Describe("Writer", func() {
|
|
17
20
|
compressed = bfs.NewInMemObject("path/to/file.jsonz")
|
18
21
|
})
|
19
22
|
|
23
|
+
It("should write plain", func() {
|
24
|
+
w, err := feedx.NewWriter(context.Background(), plain, &feedx.WriterOptions{
|
25
|
+
LastMod: time.Unix(1515151515, 123456789),
|
26
|
+
})
|
27
|
+
Expect(err).NotTo(HaveOccurred())
|
28
|
+
defer w.Discard()
|
29
|
+
|
30
|
+
Expect(w.Write(bytes.Repeat([]byte{'x'}, 10000))).To(Equal(10000))
|
31
|
+
Expect(w.Commit()).To(Succeed())
|
32
|
+
|
33
|
+
info, err := plain.Head(ctx)
|
34
|
+
Expect(err).NotTo(HaveOccurred())
|
35
|
+
Expect(info.Size).To(Equal(int64(10000)))
|
36
|
+
Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
|
37
|
+
})
|
38
|
+
|
39
|
+
It("should write compressed", func() {
|
40
|
+
w, err := feedx.NewWriter(context.Background(), compressed, &feedx.WriterOptions{
|
41
|
+
LastMod: time.Unix(1515151515, 123456789),
|
42
|
+
})
|
43
|
+
Expect(err).NotTo(HaveOccurred())
|
44
|
+
defer w.Discard()
|
45
|
+
|
46
|
+
Expect(w.Write(bytes.Repeat([]byte{'x'}, 10000))).To(Equal(10000))
|
47
|
+
Expect(w.Commit()).To(Succeed())
|
48
|
+
|
49
|
+
info, err := compressed.Head(ctx)
|
50
|
+
Expect(err).NotTo(HaveOccurred())
|
51
|
+
Expect(info.Size).To(BeNumerically("~", 50, 20))
|
52
|
+
Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
|
53
|
+
})
|
54
|
+
|
20
55
|
It("should encode", func() {
|
21
56
|
Expect(writeMulti(plain, 10)).To(Succeed())
|
22
57
|
Expect(writeMulti(compressed, 10)).To(Succeed())
|
@@ -24,11 +59,11 @@ var _ = Describe("Writer", func() {
|
|
24
59
|
info, err := plain.Head(ctx)
|
25
60
|
Expect(err).NotTo(HaveOccurred())
|
26
61
|
Expect(info.Size).To(BeNumerically("~", 470, 10))
|
27
|
-
Expect(info.Metadata).To(Equal(
|
62
|
+
Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
|
28
63
|
|
29
64
|
info, err = compressed.Head(ctx)
|
30
65
|
Expect(err).NotTo(HaveOccurred())
|
31
66
|
Expect(info.Size).To(BeNumerically("~", 76, 10))
|
32
|
-
Expect(info.Metadata).To(Equal(
|
67
|
+
Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
|
33
68
|
})
|
34
69
|
})
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: feedx
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Black Square Media Ltd
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-10-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bfs
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 0.
|
19
|
+
version: 0.5.0
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 0.
|
26
|
+
version: 0.5.0
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: bundler
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|