feedx 0.5.0 → 0.5.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/.travis.yml +6 -0
- data/Gemfile.lock +14 -13
- data/feedx.gemspec +3 -2
- data/format.go +6 -2
- data/go.mod +7 -9
- data/go.sum +106 -47
- data/producer.go +166 -0
- data/producer_test.go +43 -0
- data/spec/feedx/producer_spec.rb +1 -1
- metadata +20 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: aa8c1f3e04b9d7dceefe3eebc2e531249e0991d703bfbabcc086af20db5a9621
|
4
|
+
data.tar.gz: 5b9693716fec0f535045906e86df3af1d9f17e26b460c8cb27f33f2f446c8401
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f1238367ec92ed6237063cb31a537793872e9af672d6158c9e8b2ebb72a80cddb75ac7c44944debe383f1990fea4e0c154ee2954cefba7bff83154f761751119
|
7
|
+
data.tar.gz: 2db446f6eee8af5b334b5b81c650c1b344ec1e917cab5ce4e518896ae34706252f0b5ea081ca0bcef94de93c6bc519e278f0a8dc91b0ac315237acc7d24277e6
|
data/.rubocop.yml
CHANGED
data/.travis.yml
CHANGED
@@ -3,12 +3,18 @@ matrix:
|
|
3
3
|
- language: ruby
|
4
4
|
rvm:
|
5
5
|
- 2.6
|
6
|
+
before_install:
|
7
|
+
- gem install bundler
|
6
8
|
- language: ruby
|
7
9
|
rvm:
|
8
10
|
- 2.5
|
11
|
+
before_install:
|
12
|
+
- gem install bundler
|
9
13
|
- language: ruby
|
10
14
|
rvm:
|
11
15
|
- 2.4
|
16
|
+
before_install:
|
17
|
+
- gem install bundler
|
12
18
|
- language: go
|
13
19
|
go:
|
14
20
|
- 1.11.x
|
data/Gemfile.lock
CHANGED
@@ -1,23 +1,22 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
feedx (0.5.
|
4
|
+
feedx (0.5.1)
|
5
5
|
bfs (>= 0.3.4)
|
6
6
|
|
7
7
|
GEM
|
8
8
|
remote: https://rubygems.org/
|
9
9
|
specs:
|
10
10
|
ast (2.4.0)
|
11
|
-
bfs (0.3.
|
11
|
+
bfs (0.3.7)
|
12
12
|
diff-lcs (1.3)
|
13
|
-
google-protobuf (3.
|
13
|
+
google-protobuf (3.8.0.rc.1)
|
14
14
|
jaro_winkler (1.5.2)
|
15
|
-
parallel (1.
|
16
|
-
parser (2.6.
|
15
|
+
parallel (1.17.0)
|
16
|
+
parser (2.6.3.0)
|
17
17
|
ast (~> 2.4.0)
|
18
18
|
pbio (0.1.0)
|
19
19
|
google-protobuf
|
20
|
-
powerpack (0.1.2)
|
21
20
|
rainbow (3.0.0)
|
22
21
|
rake (12.3.2)
|
23
22
|
rspec (3.8.0)
|
@@ -26,23 +25,24 @@ GEM
|
|
26
25
|
rspec-mocks (~> 3.8.0)
|
27
26
|
rspec-core (3.8.0)
|
28
27
|
rspec-support (~> 3.8.0)
|
29
|
-
rspec-expectations (3.8.
|
28
|
+
rspec-expectations (3.8.3)
|
30
29
|
diff-lcs (>= 1.2.0, < 2.0)
|
31
30
|
rspec-support (~> 3.8.0)
|
32
31
|
rspec-mocks (3.8.0)
|
33
32
|
diff-lcs (>= 1.2.0, < 2.0)
|
34
33
|
rspec-support (~> 3.8.0)
|
35
34
|
rspec-support (3.8.0)
|
36
|
-
rubocop (0.
|
35
|
+
rubocop (0.69.0)
|
37
36
|
jaro_winkler (~> 1.5.1)
|
38
37
|
parallel (~> 1.10)
|
39
|
-
parser (>= 2.
|
40
|
-
powerpack (~> 0.1)
|
38
|
+
parser (>= 2.6)
|
41
39
|
rainbow (>= 2.2.2, < 4.0)
|
42
40
|
ruby-progressbar (~> 1.7)
|
43
|
-
unicode-display_width (
|
41
|
+
unicode-display_width (>= 1.4.0, < 1.7)
|
42
|
+
rubocop-performance (1.3.0)
|
43
|
+
rubocop (>= 0.68.0)
|
44
44
|
ruby-progressbar (1.10.0)
|
45
|
-
unicode-display_width (1.
|
45
|
+
unicode-display_width (1.6.0)
|
46
46
|
|
47
47
|
PLATFORMS
|
48
48
|
ruby
|
@@ -55,6 +55,7 @@ DEPENDENCIES
|
|
55
55
|
rake
|
56
56
|
rspec
|
57
57
|
rubocop
|
58
|
+
rubocop-performance
|
58
59
|
|
59
60
|
BUNDLED WITH
|
60
|
-
|
61
|
+
2.0.1
|
data/feedx.gemspec
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'feedx'
|
3
|
-
s.version = '0.5.
|
3
|
+
s.version = '0.5.1'
|
4
4
|
s.authors = ['Black Square Media Ltd']
|
5
5
|
s.email = ['info@blacksquaremedia.com']
|
6
6
|
s.summary = %(Exchange data between components via feeds)
|
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
|
|
11
11
|
s.files = `git ls-files -z`.split("\x0").reject {|f| f.match(%r{^spec/}) }
|
12
12
|
s.test_files = `git ls-files -z -- spec/*`.split("\x0")
|
13
13
|
s.require_paths = ['lib']
|
14
|
-
s.required_ruby_version = '>= 2.
|
14
|
+
s.required_ruby_version = '>= 2.3'
|
15
15
|
|
16
16
|
s.add_dependency 'bfs', '>= 0.3.4'
|
17
17
|
|
@@ -20,4 +20,5 @@ Gem::Specification.new do |s|
|
|
20
20
|
s.add_development_dependency 'rake'
|
21
21
|
s.add_development_dependency 'rspec'
|
22
22
|
s.add_development_dependency 'rubocop'
|
23
|
+
s.add_development_dependency 'rubocop-performance'
|
23
24
|
end
|
data/format.go
CHANGED
@@ -47,11 +47,15 @@ type FormatDecoder interface {
|
|
47
47
|
io.Closer
|
48
48
|
}
|
49
49
|
|
50
|
-
//
|
51
|
-
type
|
50
|
+
// FormatPureEncoder methods
|
51
|
+
type FormatPureEncoder interface {
|
52
52
|
// Encode encodes the value to the stream.
|
53
53
|
Encode(v interface{}) error
|
54
|
+
}
|
54
55
|
|
56
|
+
// FormatEncoder methods
|
57
|
+
type FormatEncoder interface {
|
58
|
+
FormatPureEncoder
|
55
59
|
io.Closer
|
56
60
|
}
|
57
61
|
|
data/go.mod
CHANGED
@@ -1,13 +1,11 @@
|
|
1
1
|
module github.com/bsm/feedx
|
2
2
|
|
3
3
|
require (
|
4
|
-
github.com/bsm/bfs v0.
|
5
|
-
github.com/gogo/protobuf v1.2.
|
6
|
-
github.com/golang/protobuf v1.
|
7
|
-
github.com/
|
8
|
-
github.com/onsi/
|
9
|
-
|
10
|
-
golang.org/x/
|
11
|
-
golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06 // indirect
|
12
|
-
gopkg.in/yaml.v2 v2.2.2 // indirect
|
4
|
+
github.com/bsm/bfs v0.6.1
|
5
|
+
github.com/gogo/protobuf v1.2.1
|
6
|
+
github.com/golang/protobuf v1.3.1
|
7
|
+
github.com/onsi/ginkgo v1.8.0
|
8
|
+
github.com/onsi/gomega v1.5.0
|
9
|
+
golang.org/x/net v0.0.0-20190514140710-3ec191127204 // indirect
|
10
|
+
golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f // indirect
|
13
11
|
)
|
data/go.sum
CHANGED
@@ -1,106 +1,164 @@
|
|
1
1
|
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
2
|
-
cloud.google.com/go v0.
|
3
|
-
|
4
|
-
|
5
|
-
github.com/
|
2
|
+
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
3
|
+
cloud.google.com/go v0.37.4 h1:glPeL3BQJsbF6aIIYfZizMwc5LTYz250bDMjttbBGAU=
|
4
|
+
cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw=
|
5
|
+
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
6
|
+
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
|
7
|
+
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
|
8
|
+
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
9
|
+
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
10
|
+
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
|
11
|
+
github.com/aws/aws-sdk-go v1.19.19/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
6
12
|
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
7
13
|
github.com/bmatcuk/doublestar v1.1.1 h1:YroD6BJCZBYx06yYFEWvUuKVWQn3vLLQAVmDmvTSaiQ=
|
8
14
|
github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
|
9
|
-
github.com/bsm/bfs v0.
|
10
|
-
github.com/bsm/bfs v0.
|
15
|
+
github.com/bsm/bfs v0.6.1 h1:cGw3SN8Hkd2QhOo2H6bt5rIvRWMXyjHSNlaTlM/DM0c=
|
16
|
+
github.com/bsm/bfs v0.6.1/go.mod h1:ZSsoElJoxz3gZ6iz5DgZiu735MOi2siwSDjtBNpNK24=
|
11
17
|
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
18
|
+
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
12
19
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
20
|
+
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
|
21
|
+
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
|
22
|
+
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
|
13
23
|
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
14
24
|
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
15
|
-
github.com/
|
16
|
-
github.com/go-
|
25
|
+
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
26
|
+
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
27
|
+
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
28
|
+
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
17
29
|
github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI=
|
18
30
|
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
31
|
+
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
|
32
|
+
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
|
19
33
|
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
20
|
-
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
|
21
|
-
github.com/golang/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
|
22
34
|
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
35
|
+
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
23
36
|
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
|
24
37
|
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
38
|
+
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
|
39
|
+
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
40
|
+
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
41
|
+
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
25
42
|
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
26
43
|
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
27
|
-
github.com/
|
28
|
-
github.com/
|
29
|
-
github.com/
|
30
|
-
github.com/
|
44
|
+
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
45
|
+
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
46
|
+
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
|
47
|
+
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
|
48
|
+
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
49
|
+
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
31
50
|
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
32
51
|
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
33
|
-
github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
34
52
|
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
35
|
-
github.com/
|
53
|
+
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
54
|
+
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
55
|
+
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
36
56
|
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
57
|
+
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
58
|
+
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
37
59
|
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
38
60
|
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
39
61
|
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
40
62
|
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
41
63
|
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
42
64
|
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
65
|
+
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
43
66
|
github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
|
44
67
|
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
45
68
|
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
|
46
69
|
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
47
|
-
github.com/onsi/
|
48
|
-
github.com/onsi/
|
70
|
+
github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
|
71
|
+
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
49
72
|
github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
|
50
73
|
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
51
|
-
github.com/
|
52
|
-
github.com/
|
74
|
+
github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
|
75
|
+
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
76
|
+
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
|
77
|
+
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
78
|
+
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
53
79
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
54
|
-
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
55
80
|
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
81
|
+
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
|
56
82
|
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
57
|
-
github.com/prometheus/
|
58
|
-
github.com/prometheus/common v0.
|
59
|
-
github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
83
|
+
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
84
|
+
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
60
85
|
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
61
|
-
github.com/
|
62
|
-
github.com/
|
86
|
+
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
87
|
+
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
88
|
+
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
89
|
+
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
90
|
+
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
63
91
|
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
64
|
-
|
65
|
-
|
92
|
+
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
93
|
+
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
|
94
|
+
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
95
|
+
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
96
|
+
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
97
|
+
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
66
98
|
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
99
|
+
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
100
|
+
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
101
|
+
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
67
102
|
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
68
103
|
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
69
104
|
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
70
|
-
golang.org/x/net v0.0.0-
|
71
|
-
golang.org/x/net v0.0.0-
|
72
|
-
golang.org/x/net v0.0.0-
|
73
|
-
golang.org/x/net v0.0.0-
|
105
|
+
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
106
|
+
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
107
|
+
golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
108
|
+
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
109
|
+
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
110
|
+
golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 h1:FP8hkuE6yUEaJnK7O2eTuejKWwW+Rhfj80dQ2JcKxCU=
|
111
|
+
golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
112
|
+
golang.org/x/net v0.0.0-20190514140710-3ec191127204 h1:4yG6GqBtw9C+UrLp6s2wtSniayy/Vd/3F7ffLE427XI=
|
113
|
+
golang.org/x/net v0.0.0-20190514140710-3ec191127204/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
74
114
|
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
75
|
-
golang.org/x/oauth2 v0.0.0-
|
115
|
+
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
116
|
+
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
76
117
|
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
77
118
|
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
|
78
119
|
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
120
|
+
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
121
|
+
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
79
122
|
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
123
|
+
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
80
124
|
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
81
|
-
golang.org/x/sys v0.0.0-
|
82
|
-
golang.org/x/sys v0.0.0-
|
83
|
-
golang.org/x/sys v0.0.0-
|
84
|
-
golang.org/x/sys v0.0.0-
|
125
|
+
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
126
|
+
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
127
|
+
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
128
|
+
golang.org/x/sys v0.0.0-20190429094411-2cc0cad0ac78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
129
|
+
golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f h1:Xab8gg26GrI/x3RNdVhVkHHM1XLyGeRBEvz4Q5x4YW8=
|
130
|
+
golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
85
131
|
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
|
86
132
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
133
|
+
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
134
|
+
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
135
|
+
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
136
|
+
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
137
|
+
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
87
138
|
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
88
|
-
golang.org/x/tools v0.0.0-
|
89
|
-
|
90
|
-
|
139
|
+
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
140
|
+
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
141
|
+
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
142
|
+
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
143
|
+
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
144
|
+
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
|
145
|
+
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
91
146
|
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
92
|
-
google.golang.org/appengine v1.
|
147
|
+
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
148
|
+
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
93
149
|
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
94
|
-
google.golang.org/genproto v0.0.0-
|
95
|
-
google.golang.org/genproto v0.0.0-
|
96
|
-
google.golang.org/
|
97
|
-
google.golang.org/grpc v1.
|
150
|
+
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
151
|
+
google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
152
|
+
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
153
|
+
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
|
154
|
+
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
155
|
+
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
156
|
+
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
98
157
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
99
158
|
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
100
159
|
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
101
160
|
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
102
161
|
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
103
|
-
gopkg.in/ini.v1 v1.39.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
104
162
|
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
105
163
|
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
106
164
|
gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
|
@@ -108,4 +166,5 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
|
108
166
|
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
109
167
|
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
110
168
|
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
111
|
-
honnef.co/go/tools v0.0.0-
|
169
|
+
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
170
|
+
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
data/producer.go
ADDED
@@ -0,0 +1,166 @@
|
|
1
|
+
package feedx
|
2
|
+
|
3
|
+
import (
|
4
|
+
"context"
|
5
|
+
"sync/atomic"
|
6
|
+
"time"
|
7
|
+
|
8
|
+
"github.com/bsm/bfs"
|
9
|
+
)
|
10
|
+
|
11
|
+
// ProduceFunc is a callback which is run by the producer on every iteration.
|
12
|
+
type ProduceFunc func(FormatPureEncoder) error
|
13
|
+
|
14
|
+
// ProducerOptions configure the producer instance.
|
15
|
+
type ProducerOptions struct {
|
16
|
+
WriterOptions
|
17
|
+
|
18
|
+
// The interval used by producer to initiate a cycle.
|
19
|
+
// Default: 1m
|
20
|
+
Interval time.Duration
|
21
|
+
|
22
|
+
// AfterPush callbacks are triggered after each push cycle, receiving
|
23
|
+
// an error (if occurred).
|
24
|
+
AfterPush func(error)
|
25
|
+
}
|
26
|
+
|
27
|
+
func (o *ProducerOptions) norm(name string) error {
|
28
|
+
o.WriterOptions.norm(name)
|
29
|
+
if o.Interval <= 0 {
|
30
|
+
o.Interval = time.Minute
|
31
|
+
}
|
32
|
+
return nil
|
33
|
+
}
|
34
|
+
|
35
|
+
// Producer (continously) produces a feed.
|
36
|
+
type Producer struct {
|
37
|
+
remote *bfs.Object
|
38
|
+
ownRemote bool
|
39
|
+
|
40
|
+
opt ProducerOptions
|
41
|
+
ctx context.Context
|
42
|
+
stop context.CancelFunc
|
43
|
+
pfn ProduceFunc
|
44
|
+
|
45
|
+
numWritten, lastPush, lastMod int64
|
46
|
+
}
|
47
|
+
|
48
|
+
// NewProducer inits a new feed producer.
|
49
|
+
func NewProducer(ctx context.Context, remoteURL string, opt *ProducerOptions, pfn ProduceFunc) (*Producer, error) {
|
50
|
+
remote, err := bfs.NewObject(ctx, remoteURL)
|
51
|
+
if err != nil {
|
52
|
+
return nil, err
|
53
|
+
}
|
54
|
+
|
55
|
+
p, err := NewProducerForRemote(ctx, remote, opt, pfn)
|
56
|
+
if err != nil {
|
57
|
+
_ = remote.Close()
|
58
|
+
return nil, err
|
59
|
+
}
|
60
|
+
p.ownRemote = true
|
61
|
+
return p, nil
|
62
|
+
}
|
63
|
+
|
64
|
+
// NewProducerForRemote starts a new feed producer with a remote.
|
65
|
+
func NewProducerForRemote(ctx context.Context, remote *bfs.Object, opt *ProducerOptions, pfn ProduceFunc) (*Producer, error) {
|
66
|
+
var o ProducerOptions
|
67
|
+
if opt != nil {
|
68
|
+
o = *opt
|
69
|
+
}
|
70
|
+
if err := o.norm(remote.Name()); err != nil {
|
71
|
+
return nil, err
|
72
|
+
}
|
73
|
+
|
74
|
+
ctx, stop := context.WithCancel(ctx)
|
75
|
+
p := &Producer{
|
76
|
+
remote: remote,
|
77
|
+
opt: o,
|
78
|
+
pfn: pfn,
|
79
|
+
ctx: ctx,
|
80
|
+
stop: stop,
|
81
|
+
}
|
82
|
+
|
83
|
+
// run initial push
|
84
|
+
if err := p.push(); err != nil {
|
85
|
+
_ = p.Close()
|
86
|
+
return nil, err
|
87
|
+
}
|
88
|
+
|
89
|
+
// start continuous loop
|
90
|
+
go p.loop()
|
91
|
+
|
92
|
+
return p, nil
|
93
|
+
}
|
94
|
+
|
95
|
+
// LastPush returns time of last push attempt.
|
96
|
+
func (p *Producer) LastPush() time.Time {
|
97
|
+
return timestamp(atomic.LoadInt64(&p.lastPush)).Time()
|
98
|
+
}
|
99
|
+
|
100
|
+
// LastModified returns time at which the remote feed was last modified.
|
101
|
+
func (p *Producer) LastModified() time.Time {
|
102
|
+
return timestamp(atomic.LoadInt64(&p.lastMod)).Time()
|
103
|
+
}
|
104
|
+
|
105
|
+
// NumWritten returns the number of values produced during the last push.
|
106
|
+
func (p *Producer) NumWritten() int {
|
107
|
+
return int(atomic.LoadInt64(&p.numWritten))
|
108
|
+
}
|
109
|
+
|
110
|
+
// Close stops the producer.
|
111
|
+
func (p *Producer) Close() error {
|
112
|
+
p.stop()
|
113
|
+
if p.ownRemote {
|
114
|
+
return p.remote.Close()
|
115
|
+
}
|
116
|
+
return nil
|
117
|
+
}
|
118
|
+
|
119
|
+
func (p *Producer) push() error {
|
120
|
+
var now int64
|
121
|
+
defer func() {
|
122
|
+
if now == 0 {
|
123
|
+
now = timestampFromTime(time.Now()).Millis()
|
124
|
+
}
|
125
|
+
atomic.StoreInt64(&p.lastPush, now)
|
126
|
+
}()
|
127
|
+
|
128
|
+
writer, err := NewWriter(p.ctx, p.remote, &p.opt.WriterOptions)
|
129
|
+
if err != nil {
|
130
|
+
return err
|
131
|
+
}
|
132
|
+
defer writer.Discard()
|
133
|
+
|
134
|
+
if err := p.pfn(writer); err != nil {
|
135
|
+
return err
|
136
|
+
}
|
137
|
+
|
138
|
+
if writer.NumWritten() == 0 {
|
139
|
+
return nil
|
140
|
+
}
|
141
|
+
|
142
|
+
if err := writer.Commit(); err != nil {
|
143
|
+
return err
|
144
|
+
}
|
145
|
+
|
146
|
+
now = timestampFromTime(time.Now()).Millis()
|
147
|
+
atomic.StoreInt64(&p.numWritten, int64(writer.NumWritten()))
|
148
|
+
atomic.StoreInt64(&p.lastMod, now)
|
149
|
+
return nil
|
150
|
+
}
|
151
|
+
|
152
|
+
func (p *Producer) loop() {
|
153
|
+
ticker := time.NewTicker(p.opt.Interval)
|
154
|
+
defer ticker.Stop()
|
155
|
+
|
156
|
+
for {
|
157
|
+
select {
|
158
|
+
case <-p.ctx.Done():
|
159
|
+
return
|
160
|
+
case <-ticker.C:
|
161
|
+
if err := p.push(); p.opt.AfterPush != nil {
|
162
|
+
p.opt.AfterPush(err)
|
163
|
+
}
|
164
|
+
}
|
165
|
+
}
|
166
|
+
}
|
data/producer_test.go
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
package feedx_test
|
2
|
+
|
3
|
+
import (
|
4
|
+
"context"
|
5
|
+
"time"
|
6
|
+
|
7
|
+
"github.com/bsm/bfs"
|
8
|
+
"github.com/bsm/feedx"
|
9
|
+
. "github.com/onsi/ginkgo"
|
10
|
+
. "github.com/onsi/gomega"
|
11
|
+
)
|
12
|
+
|
13
|
+
var _ = Describe("Producer", func() {
|
14
|
+
var subject *feedx.Producer
|
15
|
+
var obj *bfs.Object
|
16
|
+
var ctx = context.Background()
|
17
|
+
|
18
|
+
BeforeEach(func() {
|
19
|
+
obj = bfs.NewInMemObject("path/to/file.jsonz")
|
20
|
+
|
21
|
+
var err error
|
22
|
+
subject, err = feedx.NewProducerForRemote(ctx, obj, nil, func(enc feedx.FormatPureEncoder) error {
|
23
|
+
for i := 0; i < 10; i++ {
|
24
|
+
fix := fixture
|
25
|
+
if err := enc.Encode(&fix); err != nil {
|
26
|
+
return err
|
27
|
+
}
|
28
|
+
}
|
29
|
+
return nil
|
30
|
+
})
|
31
|
+
Expect(err).NotTo(HaveOccurred())
|
32
|
+
})
|
33
|
+
|
34
|
+
AfterEach(func() {
|
35
|
+
Expect(subject.Close()).To(Succeed())
|
36
|
+
})
|
37
|
+
|
38
|
+
It("should produce", func() {
|
39
|
+
Expect(subject.LastPush()).To(BeTemporally("~", time.Now(), time.Second))
|
40
|
+
Expect(subject.LastModified()).To(Equal(subject.LastPush()))
|
41
|
+
Expect(subject.Close()).To(Succeed())
|
42
|
+
})
|
43
|
+
})
|
data/spec/feedx/producer_spec.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: feedx
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Black Square Media Ltd
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-05-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bfs
|
@@ -94,6 +94,20 @@ dependencies:
|
|
94
94
|
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rubocop-performance
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
97
111
|
description: Use feeds to exchange data between (micro-)services.
|
98
112
|
email:
|
99
113
|
- info@blacksquaremedia.com
|
@@ -133,6 +147,8 @@ files:
|
|
133
147
|
- lib/feedx/format/protobuf.rb
|
134
148
|
- lib/feedx/producer.rb
|
135
149
|
- lib/feedx/pusher.rb
|
150
|
+
- producer.go
|
151
|
+
- producer_test.go
|
136
152
|
- reader.go
|
137
153
|
- reader_test.go
|
138
154
|
- spec/feedx/compression/gzip_spec.rb
|
@@ -157,14 +173,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
157
173
|
requirements:
|
158
174
|
- - ">="
|
159
175
|
- !ruby/object:Gem::Version
|
160
|
-
version: '2.
|
176
|
+
version: '2.3'
|
161
177
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
162
178
|
requirements:
|
163
179
|
- - ">="
|
164
180
|
- !ruby/object:Gem::Version
|
165
181
|
version: '0'
|
166
182
|
requirements: []
|
167
|
-
rubygems_version: 3.0.
|
183
|
+
rubygems_version: 3.0.3
|
168
184
|
signing_key:
|
169
185
|
specification_version: 4
|
170
186
|
summary: Exchange data between components via feeds
|