feedx 0.12.0 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +4 -4
- data/feedx.gemspec +1 -1
- data/lib/feedx/consumer.rb +9 -6
- data/lib/feedx/producer.rb +10 -7
- data/lib/feedx/stream.rb +19 -0
- data/spec/feedx/stream_spec.rb +8 -4
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 71dca5fd53dd7fe5f8a9db7222e0cd50deaa161247fb1648055f8c1475097f37
|
4
|
+
data.tar.gz: bd5d2840072c36370b2125f8decaccb2eb25fde22af4844641aff21ba6a1d265
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b41727d4151602a97112c32874084b2486c72329cb4eee74f4dd6736176e0b925dac9071fe427d04f4ed657eac5748e42102cb05d558b5bf6c1ffc465709b7e1
|
7
|
+
data.tar.gz: ae9d45a9a91f5438a1ccf39f1f1470042955af79399d9cceec60192b033ebcf617aa30b7d19e1db24aba9f697d82234390404f7540c64e3116e57bc6fd4d39e0
|
data/Gemfile.lock
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
feedx (0.12.
|
4
|
+
feedx (0.12.1)
|
5
5
|
bfs (>= 0.5.0)
|
6
6
|
|
7
7
|
GEM
|
8
8
|
remote: https://rubygems.org/
|
9
9
|
specs:
|
10
10
|
ast (2.4.1)
|
11
|
-
bfs (0.7.
|
12
|
-
diff-lcs (1.4)
|
11
|
+
bfs (0.7.1)
|
12
|
+
diff-lcs (1.4.4)
|
13
13
|
extpp (0.0.8)
|
14
14
|
gio2 (3.4.3)
|
15
15
|
gobject-introspection (= 3.4.3)
|
@@ -59,7 +59,7 @@ GEM
|
|
59
59
|
rubocop-ast (>= 0.0.3, < 1.0)
|
60
60
|
ruby-progressbar (~> 1.7)
|
61
61
|
unicode-display_width (>= 1.4.0, < 2.0)
|
62
|
-
rubocop-ast (0.0
|
62
|
+
rubocop-ast (0.1.0)
|
63
63
|
parser (>= 2.7.0.1)
|
64
64
|
rubocop-performance (1.6.1)
|
65
65
|
rubocop (>= 0.71.0)
|
data/feedx.gemspec
CHANGED
data/lib/feedx/consumer.rb
CHANGED
@@ -19,10 +19,10 @@ module Feedx
|
|
19
19
|
# @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
|
20
20
|
# @option opts [Feedx::Cache::Value] :cache cache value to store remote last modified time and consume conditionally.
|
21
21
|
def initialize(url, klass, format_options: {}, cache: nil, **opts)
|
22
|
-
@klass
|
23
|
-
@
|
24
|
-
@
|
25
|
-
@
|
22
|
+
@klass = klass
|
23
|
+
@url = url
|
24
|
+
@opts = opts.merge(format_options)
|
25
|
+
@cache = cache
|
26
26
|
|
27
27
|
return if format_options.empty? || (defined?(Gem::Deprecate) && Gem::Deprecate.skip)
|
28
28
|
|
@@ -31,21 +31,24 @@ module Feedx
|
|
31
31
|
|
32
32
|
# @return [Boolean] returns true if performed.
|
33
33
|
def each(&block)
|
34
|
+
stream = Feedx::Stream.new(@url, **@opts)
|
34
35
|
remote_rev = nil
|
35
36
|
|
36
37
|
if @cache
|
37
|
-
metadata =
|
38
|
+
metadata = stream.blob.info.metadata
|
38
39
|
local_rev = @cache.read.to_i
|
39
40
|
remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
|
40
41
|
return false if remote_rev.positive? && remote_rev <= local_rev
|
41
42
|
end
|
42
43
|
|
43
|
-
|
44
|
+
stream.open do |fmt|
|
44
45
|
fmt.decode_each(@klass, **@opts, &block)
|
45
46
|
end
|
46
47
|
@cache.write(remote_rev) if @cache && remote_rev
|
47
48
|
|
48
49
|
true
|
50
|
+
ensure
|
51
|
+
stream&.close
|
49
52
|
end
|
50
53
|
end
|
51
54
|
end
|
data/lib/feedx/producer.rb
CHANGED
@@ -22,9 +22,9 @@ module Feedx
|
|
22
22
|
@enum = enum || block
|
23
23
|
raise ArgumentError, "#{self.class.name}.new expects an :enum option or a block factory" unless @enum
|
24
24
|
|
25
|
-
@
|
25
|
+
@url = url
|
26
|
+
@opts = opts.merge(format_options)
|
26
27
|
@last_mod = last_modified
|
27
|
-
@opts = opts.merge(format_options)
|
28
28
|
|
29
29
|
return if format_options.empty? || (defined?(Gem::Deprecate) && Gem::Deprecate.skip)
|
30
30
|
|
@@ -32,23 +32,26 @@ module Feedx
|
|
32
32
|
end
|
33
33
|
|
34
34
|
def perform
|
35
|
-
|
36
|
-
|
35
|
+
stream = Feedx::Stream.new(@url, **@opts)
|
36
|
+
enum = @enum.is_a?(Proc) ? @enum.call : @enum
|
37
|
+
last_mod = @last_mod.is_a?(Proc) ? @last_mod.call(enum) : @last_mod
|
37
38
|
local_rev = last_mod.is_a?(Integer) ? last_mod : (last_mod.to_f * 1000).floor
|
38
39
|
|
39
40
|
begin
|
40
|
-
metadata =
|
41
|
+
metadata = stream.blob.info.metadata
|
41
42
|
remote_rev = (metadata[META_LAST_MODIFIED] || metadata[META_LAST_MODIFIED_DC]).to_i
|
42
43
|
return -1 unless local_rev > remote_rev
|
43
44
|
rescue BFS::FileNotFound
|
44
45
|
nil
|
45
46
|
end if local_rev.positive?
|
46
47
|
|
47
|
-
|
48
|
+
stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|
|
48
49
|
iter = enum.respond_to?(:find_each) ? :find_each : :each
|
49
50
|
enum.send(iter) {|rec| fmt.encode(rec, **@opts) }
|
50
51
|
end
|
51
|
-
|
52
|
+
stream.blob.info.size
|
53
|
+
ensure
|
54
|
+
stream&.close
|
52
55
|
end
|
53
56
|
end
|
54
57
|
end
|
data/lib/feedx/stream.rb
CHANGED
@@ -6,6 +6,18 @@ module Feedx
|
|
6
6
|
class Stream
|
7
7
|
attr_reader :blob
|
8
8
|
|
9
|
+
# Behaves like new, but accepts an optional block.
|
10
|
+
# If a block is given, streams are automatically closed after the block is yielded.
|
11
|
+
def self.open(url, **opts)
|
12
|
+
stream = new(url, **opts)
|
13
|
+
begin
|
14
|
+
yield stream
|
15
|
+
ensure
|
16
|
+
stream.close
|
17
|
+
end if block_given?
|
18
|
+
stream
|
19
|
+
end
|
20
|
+
|
9
21
|
# @param [String] url the blob URL.
|
10
22
|
# @param [Hash] opts options
|
11
23
|
# @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
|
@@ -15,6 +27,8 @@ module Feedx
|
|
15
27
|
@format = detect_format(format)
|
16
28
|
@compress = detect_compress(compress)
|
17
29
|
@opts = opts
|
30
|
+
|
31
|
+
BFS.defer(self, :close)
|
18
32
|
end
|
19
33
|
|
20
34
|
# Opens the remote for reading.
|
@@ -45,6 +59,11 @@ module Feedx
|
|
45
59
|
end
|
46
60
|
end
|
47
61
|
|
62
|
+
# Closes the underlying connection.
|
63
|
+
def close
|
64
|
+
@blob.close
|
65
|
+
end
|
66
|
+
|
48
67
|
private
|
49
68
|
|
50
69
|
def detect_format(val)
|
data/spec/feedx/stream_spec.rb
CHANGED
@@ -2,10 +2,13 @@ require 'spec_helper'
|
|
2
2
|
|
3
3
|
RSpec.describe Feedx::Stream do
|
4
4
|
let(:bucket) { BFS::Bucket::InMem.new }
|
5
|
-
|
5
|
+
let(:compressed) { described_class.new('mock:///dir/file.json.gz') }
|
6
6
|
|
7
7
|
subject { described_class.new('mock:///dir/file.json') }
|
8
|
-
|
8
|
+
|
9
|
+
before { allow(BFS).to receive(:resolve).and_return(bucket) }
|
10
|
+
after { subject.close }
|
11
|
+
after { compressed.close }
|
9
12
|
|
10
13
|
it 'should reject invalid inputs' do
|
11
14
|
expect do
|
@@ -24,8 +27,9 @@ RSpec.describe Feedx::Stream do
|
|
24
27
|
end
|
25
28
|
end
|
26
29
|
|
27
|
-
|
28
|
-
|
30
|
+
described_class.open('mock:///dir/file.txt', format: format.new) do |stream|
|
31
|
+
stream.create {|s| s.encode Feedx::TestCase::Model.new('X') }
|
32
|
+
end
|
29
33
|
|
30
34
|
expect(bucket.read('dir/file.txt')).to eq(
|
31
35
|
%({"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}\n),
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: feedx
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.12.
|
4
|
+
version: 0.12.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Black Square Media Ltd
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-07-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bfs
|