feedx 0.12.2 → 0.12.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.editorconfig +3 -0
- data/.gitignore +1 -0
- data/.travis.yml +2 -2
- data/Gemfile +0 -2
- data/Gemfile.lock +34 -34
- data/Makefile +6 -6
- data/compression.go +18 -0
- data/compression_test.go +12 -0
- data/ext/parquet/decoder.go +170 -0
- data/ext/parquet/decoder_test.go +88 -0
- data/ext/parquet/go.mod +12 -0
- data/ext/parquet/go.sum +134 -0
- data/ext/parquet/parquet.go +78 -0
- data/ext/parquet/parquet_test.go +28 -0
- data/ext/parquet/reader.go +89 -0
- data/ext/parquet/testdata/alltypes_plain.parquet +0 -0
- data/ext/parquet/types.go +51 -0
- data/feedx.gemspec +3 -3
- data/format.go +45 -15
- data/format_test.go +2 -0
- data/go.mod +7 -3
- data/go.sum +52 -3
- data/internal/testdata/testdata.pb.go +176 -77
- data/lib/feedx/cache/memory.rb +1 -0
- data/lib/feedx/producer.rb +18 -19
- data/lib/feedx/stream.rb +4 -8
- data/spec/feedx/consumer_spec.rb +1 -1
- data/spec/feedx/format/json_spec.rb +5 -5
- data/spec/feedx/stream_spec.rb +19 -7
- metadata +15 -6
data/lib/feedx/producer.rb
CHANGED
@@ -32,26 +32,25 @@ module Feedx
|
|
32
32
|
end
|
33
33
|
|
34
34
|
def perform
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
35
|
+
Feedx::Stream.open(@url, **@opts) do |stream|
|
36
|
+
enum = @enum.is_a?(Proc) ? @enum.call : @enum
|
37
|
+
last_mod = @last_mod.is_a?(Proc) ? @last_mod.call(enum) : @last_mod
|
38
|
+
local_rev = last_mod.is_a?(Integer) ? last_mod : (last_mod.to_f * 1000).floor
|
39
|
+
|
40
|
+
begin
|
41
|
+
metadata = stream.blob.info.metadata
|
42
|
+
remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
|
43
|
+
return -1 unless local_rev > remote_rev
|
44
|
+
rescue BFS::FileNotFound
|
45
|
+
nil
|
46
|
+
end if local_rev.positive?
|
47
|
+
|
48
|
+
stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|
|
49
|
+
iter = enum.respond_to?(:find_each) ? :find_each : :each
|
50
|
+
enum.send(iter) {|rec| fmt.encode(rec, **@opts) }
|
51
|
+
end
|
52
|
+
stream.blob.info.size
|
51
53
|
end
|
52
|
-
stream.blob.info.size
|
53
|
-
ensure
|
54
|
-
stream&.close
|
55
54
|
end
|
56
55
|
end
|
57
56
|
end
|
data/lib/feedx/stream.rb
CHANGED
@@ -36,12 +36,10 @@ module Feedx
|
|
36
36
|
# @param [Hash] opts BFS::Blob#open options
|
37
37
|
# @yield A block over a formatted stream.
|
38
38
|
# @yieldparam [Feedx::Format::Abstract] formatted input stream.
|
39
|
-
def open(**opts)
|
39
|
+
def open(**opts, &block)
|
40
40
|
@blob.open(**opts) do |io|
|
41
41
|
@compress.reader(io, **@opts) do |cio|
|
42
|
-
@format.decoder(cio, **@opts)
|
43
|
-
yield fmt
|
44
|
-
end
|
42
|
+
@format.decoder(cio, **@opts, &block)
|
45
43
|
end
|
46
44
|
end
|
47
45
|
end
|
@@ -50,12 +48,10 @@ module Feedx
|
|
50
48
|
# @param [Hash] opts BFS::Blob#create options
|
51
49
|
# @yield A block over a formatted stream.
|
52
50
|
# @yieldparam [Feedx::Format::Abstract] formatted output stream.
|
53
|
-
def create(**opts)
|
51
|
+
def create(**opts, &block)
|
54
52
|
@blob.create(**opts) do |io|
|
55
53
|
@compress.writer(io, **@opts) do |cio|
|
56
|
-
@format.encoder(cio, **@opts)
|
57
|
-
yield fmt
|
58
|
-
end
|
54
|
+
@format.encoder(cio, **@opts, &block)
|
59
55
|
end
|
60
56
|
end
|
61
57
|
end
|
data/spec/feedx/consumer_spec.rb
CHANGED
@@ -10,11 +10,11 @@ RSpec.describe Feedx::Format::JSON do
|
|
10
10
|
enc.encode(Feedx::TestCase::Model.new('Y'))
|
11
11
|
enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
|
12
12
|
end
|
13
|
-
expect(wio.string
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
13
|
+
expect(wio.string).to eq(<<~JSON)
|
14
|
+
{"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}
|
15
|
+
{"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}
|
16
|
+
{"title":"Z"}
|
17
|
+
JSON
|
18
18
|
|
19
19
|
subject.decoder rio do |dec|
|
20
20
|
expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
|
data/spec/feedx/stream_spec.rb
CHANGED
@@ -33,9 +33,9 @@ RSpec.describe Feedx::Stream do
|
|
33
33
|
end
|
34
34
|
expect(result).to eq(21)
|
35
35
|
|
36
|
-
expect(bucket.read('dir/file.txt')).to eq(
|
37
|
-
|
38
|
-
|
36
|
+
expect(bucket.read('dir/file.txt')).to eq(<<~JSON)
|
37
|
+
{"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}
|
38
|
+
JSON
|
39
39
|
end
|
40
40
|
|
41
41
|
it 'should encode' do
|
@@ -44,10 +44,10 @@ RSpec.describe Feedx::Stream do
|
|
44
44
|
s.encode(Feedx::TestCase::Model.new('Y'))
|
45
45
|
end
|
46
46
|
|
47
|
-
expect(bucket.read('dir/file.json')).to eq(
|
48
|
-
|
49
|
-
|
50
|
-
|
47
|
+
expect(bucket.read('dir/file.json')).to eq(<<~JSON)
|
48
|
+
{"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}
|
49
|
+
{"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}
|
50
|
+
JSON
|
51
51
|
end
|
52
52
|
|
53
53
|
it 'should encode compressed' do
|
@@ -67,6 +67,18 @@ RSpec.describe Feedx::Stream do
|
|
67
67
|
expect(bucket.info('dir/file.json').metadata).to eq('X' => '5')
|
68
68
|
end
|
69
69
|
|
70
|
+
it 'should abort encode on errors (if compressed)' do
|
71
|
+
stop = RuntimeError.new('STOP')
|
72
|
+
expect do
|
73
|
+
compressed.create do |s|
|
74
|
+
s.encode(Feedx::TestCase::Model.new('X'))
|
75
|
+
raise stop
|
76
|
+
end
|
77
|
+
end.to raise_error(stop)
|
78
|
+
|
79
|
+
expect(bucket.ls('**').to_a).to be_empty
|
80
|
+
end
|
81
|
+
|
70
82
|
it 'should decode' do
|
71
83
|
subject.create do |s|
|
72
84
|
s.encode(Feedx::TestCase::Model.new('X'))
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: feedx
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.12.
|
4
|
+
version: 0.12.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Black Square Media Ltd
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-12-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bfs
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 0.
|
19
|
+
version: 0.8.0
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 0.
|
26
|
+
version: 0.8.0
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: bundler
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -72,14 +72,14 @@ dependencies:
|
|
72
72
|
requirements:
|
73
73
|
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version:
|
75
|
+
version: 1.0.0
|
76
76
|
type: :development
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
80
|
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version:
|
82
|
+
version: 1.0.0
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: rspec
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
@@ -143,6 +143,15 @@ files:
|
|
143
143
|
- compression_test.go
|
144
144
|
- consumer.go
|
145
145
|
- consumer_test.go
|
146
|
+
- ext/parquet/decoder.go
|
147
|
+
- ext/parquet/decoder_test.go
|
148
|
+
- ext/parquet/go.mod
|
149
|
+
- ext/parquet/go.sum
|
150
|
+
- ext/parquet/parquet.go
|
151
|
+
- ext/parquet/parquet_test.go
|
152
|
+
- ext/parquet/reader.go
|
153
|
+
- ext/parquet/testdata/alltypes_plain.parquet
|
154
|
+
- ext/parquet/types.go
|
146
155
|
- feedx.gemspec
|
147
156
|
- feedx.go
|
148
157
|
- feedx_test.go
|