feedx 0.11.0 → 0.12.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 5662b9773c03d14b3d7d68b8a1ad208986b95c675cbf61d445f74ebdf3b86b09
4
- data.tar.gz: 074c6e5b86975aed1b31caa5250d7132a2d6bb68af7c221f89043f5f574d0268
3
+ metadata.gz: 78c9ab1f475804897ee6610107515f598c14590878a3134f14abecad23e151ab
4
+ data.tar.gz: 89d4d70a336b47ccba2db349fdcc4283e45ddb877599ac6618ba3da173ba8547
5
5
  SHA512:
6
- metadata.gz: 3c39e851c7e9266c9b6248f6c6010d351ae9d57809c5f6d153bd6defdf4933fa6f138637e8ad6b0ee5f8cab07dec002b2995ecc0a7ff02222974a49ab1d646e3
7
- data.tar.gz: 1846a1ba29397928281d1ba4b5c683c6c4e097e2879acedc52de21d722033290e5aac72a8f001d2012643f67661aa9fd76bac27c925782b6760b2d96295e1e80
6
+ metadata.gz: 12a539f376dd128eb4eb9d26d7a31812f656c0601375eec8721c814f404326d52feaafa80c6856fc9cf7fe66f1e1e956b9c74621d3ee88f0f8a47307f2ef04c1
7
+ data.tar.gz: c1de172489a3404f3c811aa3c41bf6d00708f36e756f4db0d4e58db0adfe8aa3908189cc07a25703682c48293c747351781650582ca4f1f1e5d3ab5681237f7c
@@ -4,3 +4,5 @@ inherit_from:
4
4
 
5
5
  AllCops:
6
6
  TargetRubyVersion: "2.4"
7
+ Metrics/ParameterLists:
8
+ Max: 10
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- feedx (0.11.0)
4
+ feedx (0.12.0)
5
5
  bfs (>= 0.5.0)
6
6
 
7
7
  GEM
@@ -9,7 +9,7 @@ GEM
9
9
  specs:
10
10
  ast (2.4.1)
11
11
  bfs (0.7.0)
12
- diff-lcs (1.3)
12
+ diff-lcs (1.4)
13
13
  extpp (0.0.8)
14
14
  gio2 (3.4.3)
15
15
  gobject-introspection (= 3.4.3)
@@ -20,9 +20,9 @@ GEM
20
20
  glib2 (= 3.4.3)
21
21
  google-protobuf (3.12.2)
22
22
  native-package-installer (1.0.9)
23
- parallel (1.19.1)
24
- parser (2.7.1.3)
25
- ast (~> 2.4.0)
23
+ parallel (1.19.2)
24
+ parser (2.7.1.4)
25
+ ast (~> 2.4.1)
26
26
  pbio (0.2.1)
27
27
  google-protobuf
28
28
  pkg-config (1.4.1)
@@ -50,13 +50,13 @@ GEM
50
50
  diff-lcs (>= 1.2.0, < 2.0)
51
51
  rspec-support (~> 3.9.0)
52
52
  rspec-support (3.9.3)
53
- rubocop (0.85.1)
53
+ rubocop (0.86.0)
54
54
  parallel (~> 1.10)
55
55
  parser (>= 2.7.0.1)
56
56
  rainbow (>= 2.2.2, < 4.0)
57
57
  regexp_parser (>= 1.7)
58
58
  rexml
59
- rubocop-ast (>= 0.0.3)
59
+ rubocop-ast (>= 0.0.3, < 1.0)
60
60
  ruby-progressbar (~> 1.7)
61
61
  unicode-display_width (>= 1.4.0, < 2.0)
62
62
  rubocop-ast (0.0.3)
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'feedx'
3
- s.version = '0.11.0'
3
+ s.version = '0.12.0'
4
4
  s.authors = ['Black Square Media Ltd']
5
5
  s.email = ['info@blacksquaremedia.com']
6
6
  s.summary = %(Exchange data between components via feeds)
@@ -5,12 +5,12 @@ class Feedx::Cache::Abstract
5
5
  end
6
6
 
7
7
  # Read reads a key.
8
- def read(_key, **_opts)
8
+ def read(_key, **)
9
9
  raise 'Not implemented'
10
10
  end
11
11
 
12
12
  # Write writes a key/value pair.
13
- def write(_key, _value, **_opts)
13
+ def write(_key, _value, **)
14
14
  raise 'Not implemented'
15
15
  end
16
16
 
@@ -1,9 +1,9 @@
1
1
  class Feedx::Compression::Abstract
2
- def reader(_io, &_block)
2
+ def reader(_io, **, &_block)
3
3
  raise 'Not implemented'
4
4
  end
5
5
 
6
- def writer(_io, &_block)
6
+ def writer(_io, **, &_block)
7
7
  raise 'Not implemented'
8
8
  end
9
9
  end
@@ -1,12 +1,12 @@
1
1
  require 'zlib'
2
2
 
3
3
  class Feedx::Compression::Gzip < Feedx::Compression::Abstract
4
- def reader(io, &block)
4
+ def reader(io, **, &block)
5
5
  force_binmode(io)
6
6
  Zlib::GzipReader.wrap(io, &block)
7
7
  end
8
8
 
9
- def writer(io, &block)
9
+ def writer(io, **, &block)
10
10
  force_binmode(io)
11
11
  Zlib::GzipWriter.wrap(io, &block)
12
12
  end
@@ -1,9 +1,9 @@
1
1
  class Feedx::Compression::None < Feedx::Compression::Abstract
2
- def reader(io)
2
+ def reader(io, **)
3
3
  yield(io)
4
4
  end
5
5
 
6
- def writer(io)
6
+ def writer(io, **)
7
7
  yield(io)
8
8
  end
9
9
  end
@@ -16,14 +16,17 @@ module Feedx
16
16
  # @param [Class] klass the record class.
17
17
  # @param [Hash] opts options
18
18
  # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
19
- # @option opts [Hash] :format_options format decode options. Default: {}.
20
19
  # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
21
20
  # @option opts [Feedx::Cache::Value] :cache cache value to store remote last modified time and consume conditionally.
22
- def initialize(url, klass, **opts)
23
- @klass = klass
24
- @stream = Feedx::Stream.new(url, **opts)
25
- @fmt_opts = opts[:format_options] || {}
26
- @cache = opts[:cache]
21
+ def initialize(url, klass, format_options: {}, cache: nil, **opts)
22
+ @klass = klass
23
+ @stream = Feedx::Stream.new(url, **opts)
24
+ @cache = cache
25
+ @opts = opts.merge(format_options)
26
+
27
+ return if format_options.empty? || (defined?(Gem::Deprecate) && Gem::Deprecate.skip)
28
+
29
+ warn "WARNING: passing format_options is deprecated; pass the options inline instead (called from #{caller(2..2).first})."
27
30
  end
28
31
 
29
32
  # @return [Boolean] returns true if performed.
@@ -38,7 +41,7 @@ module Feedx
38
41
  end
39
42
 
40
43
  @stream.open do |fmt|
41
- fmt.decode_each(@klass, **@fmt_opts, &block)
44
+ fmt.decode_each(@klass, **@opts, &block)
42
45
  end
43
46
  @cache.write(remote_rev) if @cache && remote_rev
44
47
 
@@ -39,6 +39,9 @@ module Feedx
39
39
  def registry
40
40
  @registry ||= {
41
41
  'json' => :JSON,
42
+ 'jsonl' => :JSON,
43
+ 'ndjson' => :JSON,
44
+ 'parquet' => :Parquet,
42
45
  'pb' => :Protobuf,
43
46
  'proto' => :Protobuf,
44
47
  'protobuf' => :Protobuf,
@@ -14,18 +14,21 @@ module Feedx
14
14
  # @param [Hash] opts options
15
15
  # @option opts [Enumerable,ActiveRecord::Relation] :enum relation or enumerator to stream.
16
16
  # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
17
- # @option opts [Hash] :format_options format encode options. Default: {}.
18
17
  # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
19
18
  # @option opts [Time,Proc] :last_modified the last modified time, used to determine if a push is necessary.
20
19
  # @yield A block factory to generate the relation or enumerator.
21
20
  # @yieldreturn [Enumerable,ActiveRecord::Relation] the relation or enumerator to stream.
22
- def initialize(url, **opts, &block)
23
- @enum = opts[:enum] || block
21
+ def initialize(url, last_modified: nil, format_options: {}, enum: nil, **opts, &block)
22
+ @enum = enum || block
24
23
  raise ArgumentError, "#{self.class.name}.new expects an :enum option or a block factory" unless @enum
25
24
 
26
25
  @stream = Feedx::Stream.new(url, **opts)
27
- @last_mod = opts[:last_modified]
28
- @fmt_opts = opts[:format_options] || {}
26
+ @last_mod = last_modified
27
+ @opts = opts.merge(format_options)
28
+
29
+ return if format_options.empty? || (defined?(Gem::Deprecate) && Gem::Deprecate.skip)
30
+
31
+ warn "WARNING: passing format_options is deprecated; pass the options inline instead (called from #{caller(2..2).first})."
29
32
  end
30
33
 
31
34
  def perform
@@ -43,7 +46,7 @@ module Feedx
43
46
 
44
47
  @stream.create metadata: { META_LAST_MODIFIED => local_rev.to_s } do |fmt|
45
48
  iter = enum.respond_to?(:find_each) ? :find_each : :each
46
- enum.send(iter) {|rec| fmt.encode(rec, **@fmt_opts) }
49
+ enum.send(iter) {|rec| fmt.encode(rec, **@opts) }
47
50
  end
48
51
  @stream.blob.info.size
49
52
  end
@@ -10,10 +10,11 @@ module Feedx
10
10
  # @param [Hash] opts options
11
11
  # @option opts [Symbol,Class<Feedx::Format::Abstract>] :format custom formatter. Default: from file extension.
12
12
  # @option opts [Symbol,Class<Feedx::Compression::Abstract>] :compress enable compression. Default: from file extension.
13
- def initialize(url, **opts)
13
+ def initialize(url, format: nil, compress: nil, **opts)
14
14
  @blob = BFS::Blob.new(url)
15
- @format = detect_format(opts[:format])
16
- @compress = detect_compress(opts[:compress])
15
+ @format = detect_format(format)
16
+ @compress = detect_compress(compress)
17
+ @opts = opts
17
18
  end
18
19
 
19
20
  # Opens the remote for reading.
@@ -22,8 +23,8 @@ module Feedx
22
23
  # @yieldparam [Feedx::Format::Abstract] formatted input stream.
23
24
  def open(**opts)
24
25
  @blob.open(**opts) do |io|
25
- @compress.reader(io) do |cio|
26
- @format.decoder(cio) do |fmt|
26
+ @compress.reader(io, **@opts) do |cio|
27
+ @format.decoder(cio, **@opts) do |fmt|
27
28
  yield fmt
28
29
  end
29
30
  end
@@ -36,8 +37,8 @@ module Feedx
36
37
  # @yieldparam [Feedx::Format::Abstract] formatted output stream.
37
38
  def create(**opts)
38
39
  @blob.create(**opts) do |io|
39
- @compress.writer(io) do |cio|
40
- @format.encoder(cio) do |fmt|
40
+ @compress.writer(io, **@opts) do |cio|
41
+ @format.encoder(cio, **@opts) do |fmt|
41
42
  yield fmt
42
43
  end
43
44
  end
@@ -64,4 +64,10 @@ RSpec.describe Feedx::Producer do
64
64
  size = described_class.perform 'mock:///dir/file.json', last_modified: Time.at(1515151516), enum: enumerable
65
65
  expect(size).to eq(15900)
66
66
  end
67
+
68
+ it 'should accept downstream options' do
69
+ expect do
70
+ described_class.perform 'mock:///dir/file.jsonz', enum: enumerable, x: 1, y: 'v', z: true
71
+ end.not_to raise_error
72
+ end
67
73
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: feedx
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.11.0
4
+ version: 0.12.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Black Square Media Ltd
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-06-16 00:00:00.000000000 Z
11
+ date: 2020-06-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bfs