feedx 0.7.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +2 -7
- data/Gemfile.lock +21 -21
- data/README.md +1 -1
- data/consumer.go +3 -6
- data/consumer_test.go +2 -2
- data/feedx.gemspec +2 -2
- data/feedx.go +4 -4
- data/format.go +13 -7
- data/go.mod +11 -5
- data/go.sum +24 -14
- data/lib/feedx.rb +2 -1
- data/lib/feedx/consumer.rb +2 -1
- data/lib/feedx/producer.rb +2 -1
- data/producer.go +3 -6
- data/producer_test.go +3 -3
- data/reader.go +33 -24
- data/reader_test.go +8 -0
- data/spec/feedx/producer_spec.rb +1 -1
- data/spec/feedx/stream_spec.rb +1 -1
- data/writer.go +44 -29
- data/writer_test.go +37 -2
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3d61930af3b031d191151820e5fefc106e3c64d8ece5d21513d8c91e4db195fb
|
4
|
+
data.tar.gz: 005bf01fefeb8b33299063f9c4e132ee9f1237f19a8bbf35c907e78863474121
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a4027d850b2122075bbb4d3ddc776fcd1be28f2f5db3d22c57a348d76fd821630554fe3327ac486b47b9b83e72c590476a68ab12eebc373d0fac231bb7ce449f
|
7
|
+
data.tar.gz: 857a3fc8cf13142a02634363e60478abd9171ef84c0d9a61f163c5debbea0a7a8378864d0e5ce27b4bdbe9a89d4b57df930c72a1da79602929a914a7ade38032
|
data/.travis.yml
CHANGED
@@ -10,18 +10,13 @@ matrix:
|
|
10
10
|
- 2.5
|
11
11
|
before_install:
|
12
12
|
- gem install bundler
|
13
|
-
- language: ruby
|
14
|
-
rvm:
|
15
|
-
- 2.4
|
16
|
-
before_install:
|
17
|
-
- gem install bundler
|
18
13
|
- language: go
|
19
14
|
go:
|
20
|
-
- 1.
|
15
|
+
- 1.13.x
|
21
16
|
env:
|
22
17
|
- GO111MODULE=on
|
23
18
|
- language: go
|
24
19
|
go:
|
25
|
-
- 1.
|
20
|
+
- 1.12.x
|
26
21
|
env:
|
27
22
|
- GO111MODULE=on
|
data/Gemfile.lock
CHANGED
@@ -1,45 +1,45 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
feedx (0.
|
5
|
-
bfs (>= 0.
|
4
|
+
feedx (0.8.0)
|
5
|
+
bfs (>= 0.5.0)
|
6
6
|
|
7
7
|
GEM
|
8
8
|
remote: https://rubygems.org/
|
9
9
|
specs:
|
10
10
|
ast (2.4.0)
|
11
|
-
bfs (0.
|
11
|
+
bfs (0.5.0)
|
12
12
|
diff-lcs (1.3)
|
13
|
-
google-protobuf (3.
|
13
|
+
google-protobuf (3.10.0-x86_64-linux)
|
14
14
|
jaro_winkler (1.5.3)
|
15
|
-
parallel (1.
|
16
|
-
parser (2.6.
|
15
|
+
parallel (1.18.0)
|
16
|
+
parser (2.6.5.0)
|
17
17
|
ast (~> 2.4.0)
|
18
18
|
pbio (0.1.0)
|
19
19
|
google-protobuf
|
20
20
|
rainbow (3.0.0)
|
21
|
-
rake (
|
22
|
-
rspec (3.
|
23
|
-
rspec-core (~> 3.
|
24
|
-
rspec-expectations (~> 3.
|
25
|
-
rspec-mocks (~> 3.
|
26
|
-
rspec-core (3.
|
27
|
-
rspec-support (~> 3.
|
28
|
-
rspec-expectations (3.
|
21
|
+
rake (13.0.0)
|
22
|
+
rspec (3.9.0)
|
23
|
+
rspec-core (~> 3.9.0)
|
24
|
+
rspec-expectations (~> 3.9.0)
|
25
|
+
rspec-mocks (~> 3.9.0)
|
26
|
+
rspec-core (3.9.0)
|
27
|
+
rspec-support (~> 3.9.0)
|
28
|
+
rspec-expectations (3.9.0)
|
29
29
|
diff-lcs (>= 1.2.0, < 2.0)
|
30
|
-
rspec-support (~> 3.
|
31
|
-
rspec-mocks (3.
|
30
|
+
rspec-support (~> 3.9.0)
|
31
|
+
rspec-mocks (3.9.0)
|
32
32
|
diff-lcs (>= 1.2.0, < 2.0)
|
33
|
-
rspec-support (~> 3.
|
34
|
-
rspec-support (3.
|
35
|
-
rubocop (0.
|
33
|
+
rspec-support (~> 3.9.0)
|
34
|
+
rspec-support (3.9.0)
|
35
|
+
rubocop (0.75.0)
|
36
36
|
jaro_winkler (~> 1.5.1)
|
37
37
|
parallel (~> 1.10)
|
38
38
|
parser (>= 2.6)
|
39
39
|
rainbow (>= 2.2.2, < 4.0)
|
40
40
|
ruby-progressbar (~> 1.7)
|
41
41
|
unicode-display_width (>= 1.4.0, < 1.7)
|
42
|
-
rubocop-performance (1.
|
42
|
+
rubocop-performance (1.5.0)
|
43
43
|
rubocop (>= 0.71.0)
|
44
44
|
ruby-progressbar (1.10.1)
|
45
45
|
unicode-display_width (1.6.0)
|
@@ -58,4 +58,4 @@ DEPENDENCIES
|
|
58
58
|
rubocop-performance
|
59
59
|
|
60
60
|
BUNDLED WITH
|
61
|
-
2.0.
|
61
|
+
2.0.2
|
data/README.md
CHANGED
data/consumer.go
CHANGED
@@ -21,12 +21,11 @@ type ConsumerOptions struct {
|
|
21
21
|
AfterSync func(*ConsumerSync, error)
|
22
22
|
}
|
23
23
|
|
24
|
-
func (o *ConsumerOptions) norm(name string)
|
24
|
+
func (o *ConsumerOptions) norm(name string) {
|
25
25
|
o.ReaderOptions.norm(name)
|
26
26
|
if o.Interval <= 0 {
|
27
27
|
o.Interval = time.Minute
|
28
28
|
}
|
29
|
-
return nil
|
30
29
|
}
|
31
30
|
|
32
31
|
// ConsumerSync contains the state of the last sync.
|
@@ -42,7 +41,7 @@ type ConsumerSync struct {
|
|
42
41
|
}
|
43
42
|
|
44
43
|
// ConsumeFunc is a parsing callback which is run by the consumer every sync interval.
|
45
|
-
type ConsumeFunc func(
|
44
|
+
type ConsumeFunc func(*Reader) (data interface{}, err error)
|
46
45
|
|
47
46
|
// Consumer manages data retrieval from a remote feed.
|
48
47
|
// It queries the feed in regular intervals, continuously retrieving new updates.
|
@@ -81,9 +80,7 @@ func NewConsumerForRemote(ctx context.Context, remote *bfs.Object, opt *Consumer
|
|
81
80
|
if opt != nil {
|
82
81
|
o = *opt
|
83
82
|
}
|
84
|
-
|
85
|
-
return nil, err
|
86
|
-
}
|
83
|
+
o.norm(remote.Name())
|
87
84
|
|
88
85
|
ctx, stop := context.WithCancel(ctx)
|
89
86
|
c := &consumer{
|
data/consumer_test.go
CHANGED
@@ -22,11 +22,11 @@ var _ = Describe("Consumer", func() {
|
|
22
22
|
Expect(writeMulti(obj, 2)).To(Succeed())
|
23
23
|
|
24
24
|
var err error
|
25
|
-
subject, err = feedx.NewConsumerForRemote(ctx, obj, nil, func(
|
25
|
+
subject, err = feedx.NewConsumerForRemote(ctx, obj, nil, func(r *feedx.Reader) (interface{}, error) {
|
26
26
|
var msgs []tbp.Message
|
27
27
|
for {
|
28
28
|
var msg tbp.Message
|
29
|
-
if err :=
|
29
|
+
if err := r.Decode(&msg); err == io.EOF {
|
30
30
|
break
|
31
31
|
}
|
32
32
|
if err != nil {
|
data/feedx.gemspec
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'feedx'
|
3
|
-
s.version = '0.
|
3
|
+
s.version = '0.8.0'
|
4
4
|
s.authors = ['Black Square Media Ltd']
|
5
5
|
s.email = ['info@blacksquaremedia.com']
|
6
6
|
s.summary = %(Exchange data between components via feeds)
|
@@ -13,7 +13,7 @@ Gem::Specification.new do |s|
|
|
13
13
|
s.require_paths = ['lib']
|
14
14
|
s.required_ruby_version = '>= 2.4'
|
15
15
|
|
16
|
-
s.add_dependency 'bfs', '>= 0.
|
16
|
+
s.add_dependency 'bfs', '>= 0.5.0'
|
17
17
|
|
18
18
|
s.add_development_dependency 'bundler'
|
19
19
|
s.add_development_dependency 'pbio'
|
data/feedx.go
CHANGED
@@ -13,8 +13,8 @@ import (
|
|
13
13
|
var ErrNotModified = errors.New("feedx: not modified")
|
14
14
|
|
15
15
|
const (
|
16
|
-
metaLastModified = "
|
17
|
-
metaPusherLastModified = "
|
16
|
+
metaLastModified = "X-Feedx-Last-Modified"
|
17
|
+
metaPusherLastModified = "X-Feedx-Pusher-Last-Modified"
|
18
18
|
)
|
19
19
|
|
20
20
|
// Timestamp with millisecond resolution
|
@@ -35,9 +35,9 @@ func remoteLastModified(ctx context.Context, obj *bfs.Object) (timestamp, error)
|
|
35
35
|
return 0, err
|
36
36
|
}
|
37
37
|
|
38
|
-
millis, _ := strconv.ParseInt(info.Metadata
|
38
|
+
millis, _ := strconv.ParseInt(info.Metadata.Get(metaLastModified), 10, 64)
|
39
39
|
if millis == 0 {
|
40
|
-
millis, _ = strconv.ParseInt(info.Metadata
|
40
|
+
millis, _ = strconv.ParseInt(info.Metadata.Get(metaPusherLastModified), 10, 64)
|
41
41
|
}
|
42
42
|
return timestamp(millis), nil
|
43
43
|
}
|
data/format.go
CHANGED
@@ -2,6 +2,7 @@ package feedx
|
|
2
2
|
|
3
3
|
import (
|
4
4
|
"encoding/json"
|
5
|
+
"errors"
|
5
6
|
"fmt"
|
6
7
|
"io"
|
7
8
|
"path"
|
@@ -11,6 +12,8 @@ import (
|
|
11
12
|
pbio "github.com/gogo/protobuf/io"
|
12
13
|
)
|
13
14
|
|
15
|
+
var errNoFormat = errors.New("feedx: no format detected")
|
16
|
+
|
14
17
|
// Format represents the data format.
|
15
18
|
type Format interface {
|
16
19
|
// NewDecoder wraps a decoder around a reader.
|
@@ -36,7 +39,7 @@ func DetectFormat(name string) Format {
|
|
36
39
|
return DetectFormat(name[0 : len(name)-len(ext)])
|
37
40
|
}
|
38
41
|
}
|
39
|
-
return nil
|
42
|
+
return (*noFormat)(nil)
|
40
43
|
}
|
41
44
|
|
42
45
|
// FormatDecoder methods
|
@@ -47,20 +50,23 @@ type FormatDecoder interface {
|
|
47
50
|
io.Closer
|
48
51
|
}
|
49
52
|
|
50
|
-
//
|
51
|
-
type
|
53
|
+
// FormatEncoder methods
|
54
|
+
type FormatEncoder interface {
|
52
55
|
// Encode encodes the value to the stream.
|
53
56
|
Encode(v interface{}) error
|
54
|
-
}
|
55
57
|
|
56
|
-
// FormatEncoder methods
|
57
|
-
type FormatEncoder interface {
|
58
|
-
FormatPureEncoder
|
59
58
|
io.Closer
|
60
59
|
}
|
61
60
|
|
62
61
|
// --------------------------------------------------------------------
|
63
62
|
|
63
|
+
type noFormat struct{}
|
64
|
+
|
65
|
+
func (*noFormat) NewDecoder(r io.Reader) (FormatDecoder, error) { return nil, errNoFormat }
|
66
|
+
func (*noFormat) NewEncoder(w io.Writer) (FormatEncoder, error) { return nil, errNoFormat }
|
67
|
+
|
68
|
+
// --------------------------------------------------------------------
|
69
|
+
|
64
70
|
// JSONFormat provides a Format implemention for JSON.
|
65
71
|
var JSONFormat = jsonFormat{}
|
66
72
|
|
data/go.mod
CHANGED
@@ -1,9 +1,15 @@
|
|
1
1
|
module github.com/bsm/feedx
|
2
2
|
|
3
3
|
require (
|
4
|
-
github.com/bsm/bfs v0.
|
5
|
-
github.com/gogo/protobuf v1.
|
6
|
-
github.com/golang/protobuf v1.3.
|
7
|
-
github.com/onsi/ginkgo v1.
|
8
|
-
github.com/onsi/gomega v1.
|
4
|
+
github.com/bsm/bfs v0.8.0
|
5
|
+
github.com/gogo/protobuf v1.3.0
|
6
|
+
github.com/golang/protobuf v1.3.2
|
7
|
+
github.com/onsi/ginkgo v1.10.2
|
8
|
+
github.com/onsi/gomega v1.7.0
|
9
|
+
golang.org/x/net v0.0.0-20191007182048-72f939374954 // indirect
|
10
|
+
golang.org/x/sys v0.0.0-20191008105621-543471e840be // indirect
|
11
|
+
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
|
12
|
+
gopkg.in/yaml.v2 v2.2.4 // indirect
|
9
13
|
)
|
14
|
+
|
15
|
+
go 1.13
|
data/go.sum
CHANGED
@@ -1,18 +1,18 @@
|
|
1
|
-
github.com/bmatcuk/doublestar v1.1.
|
2
|
-
github.com/bmatcuk/doublestar v1.1.
|
3
|
-
github.com/bsm/bfs v0.
|
4
|
-
github.com/bsm/bfs v0.
|
1
|
+
github.com/bmatcuk/doublestar v1.1.5 h1:2bNwBOmhyFEFcoB3tGvTD5xanq+4kyOZlB8wFYbMjkk=
|
2
|
+
github.com/bmatcuk/doublestar v1.1.5/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
|
3
|
+
github.com/bsm/bfs v0.8.0 h1:/suMKytZ3WhVY62osdFm6VD+gJtxaohlUBuf76vGC78=
|
4
|
+
github.com/bsm/bfs v0.8.0/go.mod h1:cVv0jyqUY/jbHoG/WYPuWvOaOhW/HZ4jl7/JMlypvAE=
|
5
5
|
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
6
6
|
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
7
|
-
github.com/gogo/protobuf v1.
|
8
|
-
github.com/gogo/protobuf v1.
|
7
|
+
github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE=
|
8
|
+
github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
|
9
9
|
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
|
10
10
|
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
11
|
-
github.com/golang/protobuf v1.3.
|
12
|
-
github.com/golang/protobuf v1.3.
|
11
|
+
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
|
12
|
+
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
13
13
|
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
14
14
|
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
15
|
-
github.com/kisielk/errcheck v1.
|
15
|
+
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
|
16
16
|
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
17
17
|
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
18
18
|
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
@@ -23,26 +23,34 @@ github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
|
|
23
23
|
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
24
24
|
github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
|
25
25
|
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
26
|
+
github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94=
|
27
|
+
github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
26
28
|
github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
|
27
29
|
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
30
|
+
github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
|
31
|
+
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
28
32
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
29
33
|
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
30
|
-
golang.org/x/net v0.0.0-
|
31
|
-
golang.org/x/net v0.0.0-
|
34
|
+
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
35
|
+
golang.org/x/net v0.0.0-20191007182048-72f939374954 h1:JGZucVF/L/TotR719NbujzadOZ2AgnYlqphQGHDCKaU=
|
36
|
+
golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
32
37
|
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
33
38
|
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
34
39
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
35
|
-
golang.org/x/sys v0.0.0-
|
36
|
-
golang.org/x/sys v0.0.0-
|
40
|
+
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
41
|
+
golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU=
|
42
|
+
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
37
43
|
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
|
38
44
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
39
45
|
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
40
46
|
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
41
|
-
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
42
47
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
48
|
+
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
43
49
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
44
50
|
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
45
51
|
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
52
|
+
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
53
|
+
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
46
54
|
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
47
55
|
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
48
56
|
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
@@ -51,3 +59,5 @@ gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
|
|
51
59
|
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
52
60
|
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
53
61
|
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
62
|
+
gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
|
63
|
+
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
data/lib/feedx.rb
CHANGED
data/lib/feedx/consumer.rb
CHANGED
@@ -31,8 +31,9 @@ module Feedx
|
|
31
31
|
remote_rev = nil
|
32
32
|
|
33
33
|
if @cache
|
34
|
+
metadata = @stream.blob.info.metadata
|
34
35
|
local_rev = @cache.read.to_i
|
35
|
-
remote_rev =
|
36
|
+
remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
|
36
37
|
return false if remote_rev.positive? && remote_rev <= local_rev
|
37
38
|
end
|
38
39
|
|
data/lib/feedx/producer.rb
CHANGED
@@ -34,7 +34,8 @@ module Feedx
|
|
34
34
|
local_rev = last_mod.is_a?(Integer) ? last_mod : (last_mod.to_f * 1000).floor
|
35
35
|
|
36
36
|
begin
|
37
|
-
|
37
|
+
metadata = @stream.blob.info.metadata
|
38
|
+
remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
|
38
39
|
return -1 unless local_rev > remote_rev
|
39
40
|
rescue BFS::FileNotFound # rubocop:disable Lint/HandleExceptions
|
40
41
|
end if local_rev.positive?
|
data/producer.go
CHANGED
@@ -9,7 +9,7 @@ import (
|
|
9
9
|
)
|
10
10
|
|
11
11
|
// ProduceFunc is a callback which is run by the producer on every iteration.
|
12
|
-
type ProduceFunc func(
|
12
|
+
type ProduceFunc func(*Writer) error
|
13
13
|
|
14
14
|
// ProducerOptions configure the producer instance.
|
15
15
|
type ProducerOptions struct {
|
@@ -28,12 +28,11 @@ type ProducerOptions struct {
|
|
28
28
|
AfterPush func(*ProducerPush, error)
|
29
29
|
}
|
30
30
|
|
31
|
-
func (o *ProducerOptions) norm(name string)
|
31
|
+
func (o *ProducerOptions) norm(name string) {
|
32
32
|
o.WriterOptions.norm(name)
|
33
33
|
if o.Interval <= 0 {
|
34
34
|
o.Interval = time.Minute
|
35
35
|
}
|
36
|
-
return nil
|
37
36
|
}
|
38
37
|
|
39
38
|
// ProducerPush contains the state of the last push.
|
@@ -79,9 +78,7 @@ func NewProducerForRemote(ctx context.Context, remote *bfs.Object, opt *Producer
|
|
79
78
|
if opt != nil {
|
80
79
|
o = *opt
|
81
80
|
}
|
82
|
-
|
83
|
-
return nil, err
|
84
|
-
}
|
81
|
+
o.norm(remote.Name())
|
85
82
|
|
86
83
|
ctx, stop := context.WithCancel(ctx)
|
87
84
|
p := &Producer{
|
data/producer_test.go
CHANGED
@@ -17,10 +17,10 @@ var _ = Describe("Producer", func() {
|
|
17
17
|
|
18
18
|
setup := func(o *feedx.ProducerOptions) {
|
19
19
|
var err error
|
20
|
-
subject, err = feedx.NewProducerForRemote(ctx, obj, o, func(
|
20
|
+
subject, err = feedx.NewProducerForRemote(ctx, obj, o, func(w *feedx.Writer) error {
|
21
21
|
for i := 0; i < 10; i++ {
|
22
22
|
fix := fixture
|
23
|
-
if err :=
|
23
|
+
if err := w.Encode(&fix); err != nil {
|
24
24
|
return err
|
25
25
|
}
|
26
26
|
}
|
@@ -64,6 +64,6 @@ var _ = Describe("Producer", func() {
|
|
64
64
|
info, err := obj.Head(ctx)
|
65
65
|
Expect(err).NotTo(HaveOccurred())
|
66
66
|
Expect(info.Size).To(BeNumerically("~", 75, 10))
|
67
|
-
Expect(info.Metadata).To(HaveKeyWithValue("
|
67
|
+
Expect(info.Metadata).To(HaveKeyWithValue("X-Feedx-Last-Modified", "1515151515000"))
|
68
68
|
})
|
69
69
|
})
|
data/reader.go
CHANGED
@@ -2,7 +2,6 @@ package feedx
|
|
2
2
|
|
3
3
|
import (
|
4
4
|
"context"
|
5
|
-
"fmt"
|
6
5
|
"io"
|
7
6
|
"time"
|
8
7
|
|
@@ -20,18 +19,13 @@ type ReaderOptions struct {
|
|
20
19
|
Compression Compression
|
21
20
|
}
|
22
21
|
|
23
|
-
func (o *ReaderOptions) norm(name string)
|
22
|
+
func (o *ReaderOptions) norm(name string) {
|
24
23
|
if o.Format == nil {
|
25
24
|
o.Format = DetectFormat(name)
|
26
|
-
|
27
|
-
if o.Format == nil {
|
28
|
-
return fmt.Errorf("feedx: unable to detect format from %q", name)
|
29
|
-
}
|
30
25
|
}
|
31
26
|
if o.Compression == nil {
|
32
27
|
o.Compression = DetectCompression(name)
|
33
28
|
}
|
34
|
-
return nil
|
35
29
|
}
|
36
30
|
|
37
31
|
// Reader reads data from a remote feed.
|
@@ -52,9 +46,7 @@ func NewReader(ctx context.Context, remote *bfs.Object, opt *ReaderOptions) (*Re
|
|
52
46
|
if opt != nil {
|
53
47
|
o = *opt
|
54
48
|
}
|
55
|
-
|
56
|
-
return nil, err
|
57
|
-
}
|
49
|
+
o.norm(remote.Name())
|
58
50
|
|
59
51
|
return &Reader{
|
60
52
|
remote: remote,
|
@@ -63,22 +55,19 @@ func NewReader(ctx context.Context, remote *bfs.Object, opt *ReaderOptions) (*Re
|
|
63
55
|
}, nil
|
64
56
|
}
|
65
57
|
|
66
|
-
//
|
67
|
-
func (r *Reader)
|
68
|
-
if r.
|
69
|
-
|
70
|
-
if err != nil {
|
71
|
-
return err
|
72
|
-
}
|
73
|
-
r.br = br
|
58
|
+
// Read reads raw bytes from the feed.
|
59
|
+
func (r *Reader) Read(p []byte) (int, error) {
|
60
|
+
if err := r.ensureOpen(); err != nil {
|
61
|
+
return 0, err
|
74
62
|
}
|
75
63
|
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
64
|
+
return r.cr.Read(p)
|
65
|
+
}
|
66
|
+
|
67
|
+
// Decode decodes the next formatted value from the feed.
|
68
|
+
func (r *Reader) Decode(v interface{}) error {
|
69
|
+
if err := r.ensureOpen(); err != nil {
|
70
|
+
return err
|
82
71
|
}
|
83
72
|
|
84
73
|
if r.fd == nil {
|
@@ -128,3 +117,23 @@ func (r *Reader) Close() error {
|
|
128
117
|
}
|
129
118
|
return err
|
130
119
|
}
|
120
|
+
|
121
|
+
func (r *Reader) ensureOpen() error {
|
122
|
+
if r.br == nil {
|
123
|
+
br, err := r.remote.Open(r.ctx)
|
124
|
+
if err != nil {
|
125
|
+
return err
|
126
|
+
}
|
127
|
+
r.br = br
|
128
|
+
}
|
129
|
+
|
130
|
+
if r.cr == nil {
|
131
|
+
cr, err := r.opt.Compression.NewReader(r.br)
|
132
|
+
if err != nil {
|
133
|
+
return err
|
134
|
+
}
|
135
|
+
r.cr = cr
|
136
|
+
}
|
137
|
+
|
138
|
+
return nil
|
139
|
+
}
|
data/reader_test.go
CHANGED
@@ -3,6 +3,7 @@ package feedx_test
|
|
3
3
|
import (
|
4
4
|
"context"
|
5
5
|
"io"
|
6
|
+
"io/ioutil"
|
6
7
|
|
7
8
|
"github.com/bsm/feedx"
|
8
9
|
|
@@ -31,6 +32,13 @@ var _ = Describe("Reader", func() {
|
|
31
32
|
})
|
32
33
|
|
33
34
|
It("should read", func() {
|
35
|
+
data, err := ioutil.ReadAll(subject)
|
36
|
+
Expect(err).NotTo(HaveOccurred())
|
37
|
+
Expect(len(data)).To(BeNumerically("~", 140, 20))
|
38
|
+
Expect(subject.NumRead()).To(Equal(0))
|
39
|
+
})
|
40
|
+
|
41
|
+
It("should decode", func() {
|
34
42
|
var msgs []tbp.Message
|
35
43
|
for {
|
36
44
|
var msg tbp.Message
|
data/spec/feedx/producer_spec.rb
CHANGED
@@ -48,7 +48,7 @@ RSpec.describe Feedx::Producer do
|
|
48
48
|
|
49
49
|
it 'should support last-modified' do
|
50
50
|
described_class.perform 'mock:///dir/file.json', last_modified: Time.at(1515151515), enum: enumerable
|
51
|
-
expect(bucket.info('dir/file.json').metadata).to eq('
|
51
|
+
expect(bucket.info('dir/file.json').metadata).to eq('X-Feedx-Last-Modified' => '1515151515000')
|
52
52
|
end
|
53
53
|
|
54
54
|
it 'should perform conditionally' do
|
data/spec/feedx/stream_spec.rb
CHANGED
@@ -39,7 +39,7 @@ RSpec.describe Feedx::Stream do
|
|
39
39
|
subject.create metadata: { 'x' => '5' } do |s|
|
40
40
|
s.encode(Feedx::TestCase::Model.new('X'))
|
41
41
|
end
|
42
|
-
expect(bucket.info('dir/file.json').metadata).to eq('
|
42
|
+
expect(bucket.info('dir/file.json').metadata).to eq('X' => '5')
|
43
43
|
end
|
44
44
|
|
45
45
|
it 'should decode' do
|
data/writer.go
CHANGED
@@ -3,7 +3,6 @@ package feedx
|
|
3
3
|
import (
|
4
4
|
"bufio"
|
5
5
|
"context"
|
6
|
-
"fmt"
|
7
6
|
"io"
|
8
7
|
"time"
|
9
8
|
|
@@ -25,24 +24,16 @@ type WriterOptions struct {
|
|
25
24
|
LastMod time.Time
|
26
25
|
}
|
27
26
|
|
28
|
-
func (o *WriterOptions) norm(name string)
|
27
|
+
func (o *WriterOptions) norm(name string) {
|
29
28
|
if o.Format == nil {
|
30
29
|
o.Format = DetectFormat(name)
|
31
|
-
|
32
|
-
if o.Format == nil {
|
33
|
-
return fmt.Errorf("feedx: unable to detect format from %q", name)
|
34
|
-
}
|
35
30
|
}
|
36
|
-
|
37
31
|
if o.Compression == nil {
|
38
32
|
o.Compression = DetectCompression(name)
|
39
33
|
}
|
40
|
-
|
41
34
|
if o.LastMod.IsZero() {
|
42
35
|
o.LastMod = time.Now()
|
43
36
|
}
|
44
|
-
|
45
|
-
return nil
|
46
37
|
}
|
47
38
|
|
48
39
|
// Writer encodes feeds to remote locations.
|
@@ -77,29 +68,26 @@ func NewWriter(ctx context.Context, remote *bfs.Object, opt *WriterOptions) (*Wr
|
|
77
68
|
}, nil
|
78
69
|
}
|
79
70
|
|
80
|
-
//
|
81
|
-
func (w *Writer)
|
82
|
-
if w.
|
83
|
-
|
84
|
-
bw, err := w.remote.Create(w.ctx, &bfs.WriteOptions{
|
85
|
-
Metadata: map[string]string{metaLastModified: ts.String()},
|
86
|
-
})
|
87
|
-
if err != nil {
|
88
|
-
return err
|
89
|
-
}
|
90
|
-
w.bw = bw
|
71
|
+
// Write write raw bytes to the feed.
|
72
|
+
func (w *Writer) Write(p []byte) (int, error) {
|
73
|
+
if err := w.ensureCreated(); err != nil {
|
74
|
+
return 0, err
|
91
75
|
}
|
76
|
+
return w.ww.Write(p)
|
77
|
+
}
|
92
78
|
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
}
|
98
|
-
w.cw = cw
|
79
|
+
// WriteString write a raw string to the feed.
|
80
|
+
func (w *Writer) WriteString(s string) (int, error) {
|
81
|
+
if err := w.ensureCreated(); err != nil {
|
82
|
+
return 0, err
|
99
83
|
}
|
84
|
+
return w.ww.WriteString(s)
|
85
|
+
}
|
100
86
|
|
101
|
-
|
102
|
-
|
87
|
+
// Encode appends a value to the feed.
|
88
|
+
func (w *Writer) Encode(v interface{}) error {
|
89
|
+
if err := w.ensureCreated(); err != nil {
|
90
|
+
return err
|
103
91
|
}
|
104
92
|
|
105
93
|
if w.fe == nil {
|
@@ -154,3 +142,30 @@ func (w *Writer) Commit() error {
|
|
154
142
|
}
|
155
143
|
return err
|
156
144
|
}
|
145
|
+
|
146
|
+
func (w *Writer) ensureCreated() error {
|
147
|
+
if w.bw == nil {
|
148
|
+
ts := timestampFromTime(w.opt.LastMod)
|
149
|
+
bw, err := w.remote.Create(w.ctx, &bfs.WriteOptions{
|
150
|
+
Metadata: bfs.Metadata{metaLastModified: ts.String()},
|
151
|
+
})
|
152
|
+
if err != nil {
|
153
|
+
return err
|
154
|
+
}
|
155
|
+
w.bw = bw
|
156
|
+
}
|
157
|
+
|
158
|
+
if w.cw == nil {
|
159
|
+
cw, err := w.opt.Compression.NewWriter(w.bw)
|
160
|
+
if err != nil {
|
161
|
+
return err
|
162
|
+
}
|
163
|
+
w.cw = cw
|
164
|
+
}
|
165
|
+
|
166
|
+
if w.ww == nil {
|
167
|
+
w.ww = bufio.NewWriter(w.cw)
|
168
|
+
}
|
169
|
+
|
170
|
+
return nil
|
171
|
+
}
|
data/writer_test.go
CHANGED
@@ -1,9 +1,12 @@
|
|
1
1
|
package feedx_test
|
2
2
|
|
3
3
|
import (
|
4
|
+
"bytes"
|
4
5
|
"context"
|
6
|
+
"time"
|
5
7
|
|
6
8
|
"github.com/bsm/bfs"
|
9
|
+
"github.com/bsm/feedx"
|
7
10
|
. "github.com/onsi/ginkgo"
|
8
11
|
. "github.com/onsi/gomega"
|
9
12
|
)
|
@@ -17,6 +20,38 @@ var _ = Describe("Writer", func() {
|
|
17
20
|
compressed = bfs.NewInMemObject("path/to/file.jsonz")
|
18
21
|
})
|
19
22
|
|
23
|
+
It("should write plain", func() {
|
24
|
+
w, err := feedx.NewWriter(context.Background(), plain, &feedx.WriterOptions{
|
25
|
+
LastMod: time.Unix(1515151515, 123456789),
|
26
|
+
})
|
27
|
+
Expect(err).NotTo(HaveOccurred())
|
28
|
+
defer w.Discard()
|
29
|
+
|
30
|
+
Expect(w.Write(bytes.Repeat([]byte{'x'}, 10000))).To(Equal(10000))
|
31
|
+
Expect(w.Commit()).To(Succeed())
|
32
|
+
|
33
|
+
info, err := plain.Head(ctx)
|
34
|
+
Expect(err).NotTo(HaveOccurred())
|
35
|
+
Expect(info.Size).To(Equal(int64(10000)))
|
36
|
+
Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
|
37
|
+
})
|
38
|
+
|
39
|
+
It("should write compressed", func() {
|
40
|
+
w, err := feedx.NewWriter(context.Background(), compressed, &feedx.WriterOptions{
|
41
|
+
LastMod: time.Unix(1515151515, 123456789),
|
42
|
+
})
|
43
|
+
Expect(err).NotTo(HaveOccurred())
|
44
|
+
defer w.Discard()
|
45
|
+
|
46
|
+
Expect(w.Write(bytes.Repeat([]byte{'x'}, 10000))).To(Equal(10000))
|
47
|
+
Expect(w.Commit()).To(Succeed())
|
48
|
+
|
49
|
+
info, err := compressed.Head(ctx)
|
50
|
+
Expect(err).NotTo(HaveOccurred())
|
51
|
+
Expect(info.Size).To(BeNumerically("~", 50, 20))
|
52
|
+
Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
|
53
|
+
})
|
54
|
+
|
20
55
|
It("should encode", func() {
|
21
56
|
Expect(writeMulti(plain, 10)).To(Succeed())
|
22
57
|
Expect(writeMulti(compressed, 10)).To(Succeed())
|
@@ -24,11 +59,11 @@ var _ = Describe("Writer", func() {
|
|
24
59
|
info, err := plain.Head(ctx)
|
25
60
|
Expect(err).NotTo(HaveOccurred())
|
26
61
|
Expect(info.Size).To(BeNumerically("~", 470, 10))
|
27
|
-
Expect(info.Metadata).To(Equal(
|
62
|
+
Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
|
28
63
|
|
29
64
|
info, err = compressed.Head(ctx)
|
30
65
|
Expect(err).NotTo(HaveOccurred())
|
31
66
|
Expect(info.Size).To(BeNumerically("~", 76, 10))
|
32
|
-
Expect(info.Metadata).To(Equal(
|
67
|
+
Expect(info.Metadata).To(Equal(bfs.Metadata{"X-Feedx-Last-Modified": "1515151515123"}))
|
33
68
|
})
|
34
69
|
})
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: feedx
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Black Square Media Ltd
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-10-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bfs
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 0.
|
19
|
+
version: 0.5.0
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 0.
|
26
|
+
version: 0.5.0
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: bundler
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|