importeur 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. checksums.yaml +5 -5
  2. data/importeur.gemspec +8 -9
  3. data/lib/importeur/active_record_postgres_loader.rb +9 -5
  4. data/lib/importeur/extractor.rb +1 -1
  5. data/lib/importeur/version.rb +1 -1
  6. metadata +44 -44
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 2a5b8cd8b68e60849d7f6fc4921b41d4ff39187a
4
- data.tar.gz: 9a23992b2d88fdad38518abc43057d3d585ea11c
2
+ SHA256:
3
+ metadata.gz: 2e2da53fbdf2d6b7e54b7bf864a0ef4585088e854225cb50b3360cc603719ca5
4
+ data.tar.gz: ab5645b728d4f8f083406d754076c00f3d5f8890e02e47e9324a30e0e55dfb53
5
5
  SHA512:
6
- metadata.gz: e1e552cb90dc22addb41d5fdaf14cb4774ebdb13952849a46be3f5d20d3d89741078a22498133039f9411e9d3866882b87b1989afedef3291e9c38c7431191ac
7
- data.tar.gz: 90056d4a61059cffe16bcd2cf2c6198a8485045568e5cfad40f009cdaff358aedc91865325e95343396c5b6b7c55682b86ca60f9736e277b38e8c7e198ed9138
6
+ metadata.gz: a81238cc10368f58201b6ed26a4234a83f3037b99baac14d4563af05a4032d8ba6e8684d13b51535d79d16969107ad14b975078d4b4cfedbcf85ea076c367f0e
7
+ data.tar.gz: d29de110bd13b0665b3ac1f62c9c08d0d443c9d45b853b281e1aa8b95f57ecc8230174ef7ad63d8a0bde4ddb865becb00bdcb00478ce93c1bd9af9e4433f03d4
data/importeur.gemspec CHANGED
@@ -1,4 +1,3 @@
1
- # coding: utf-8
2
1
  # frozen_string_literal: true
3
2
 
4
3
  lib = File.expand_path('../lib', __FILE__)
@@ -31,17 +30,17 @@ Gem::Specification.new do |spec|
31
30
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
32
31
  spec.require_paths = ['lib']
33
32
 
33
+ spec.add_development_dependency 'activerecord', '<5'
34
+ spec.add_development_dependency 'appnexusapi'
35
+ spec.add_development_dependency 'bucket_cake'
34
36
  spec.add_development_dependency 'bundler', '~> 1.15'
37
+ spec.add_development_dependency 'database_cleaner'
38
+ spec.add_development_dependency 'dotenv'
39
+ spec.add_development_dependency 'pg', '<1'
40
+ spec.add_development_dependency 'pry'
35
41
  spec.add_development_dependency 'rake', '~> 10.0'
42
+ spec.add_development_dependency 'rocketfuel_api'
36
43
  spec.add_development_dependency 'rspec', '~> 3.0'
37
- spec.add_development_dependency 'activerecord', '<5'
38
- spec.add_development_dependency 'pry'
39
44
  spec.add_development_dependency 'vcr'
40
45
  spec.add_development_dependency 'webmock'
41
- spec.add_development_dependency 'bucket_cake'
42
- spec.add_development_dependency 'pg'
43
- spec.add_development_dependency 'database_cleaner'
44
- spec.add_development_dependency 'appnexusapi'
45
- spec.add_development_dependency 'rocketfuel_api'
46
- spec.add_development_dependency 'dotenv'
47
46
  end
data/lib/importeur/active_record_postgres_loader.rb CHANGED
@@ -44,7 +44,7 @@ module Importeur
44
44
  def records_for_batch(batch_ids)
45
45
  model
46
46
  .with_deleted
47
- .where(primary_key => batch_ids)
47
+ .joins(batch_lookup_join_sql('INNER', batch_ids))
48
48
  .index_by(&primary_key)
49
49
  end
50
50
 
@@ -52,12 +52,16 @@ module Importeur
52
52
  # Basically `self.class.model.where.not(primary_key => imported_ids)`, but
53
53
  # more efficient in this case.
54
54
  model
55
- .joins(<<-SQL)
56
- LEFT JOIN (SELECT unnest(ARRAY[#{imported_ids.join(',')}]::int[]) AS primary_key) AS imported
57
- ON imported.primary_key = #{model.table_name}.#{primary_key}
58
- SQL
55
+ .joins(batch_lookup_join_sql('LEFT', imported_ids))
59
56
  .where('imported.primary_key' => nil)
60
57
  .delete_all
61
58
  end
59
+
60
+ def batch_lookup_join_sql(kind, ids)
61
+ <<-SQL
62
+ #{kind} JOIN (SELECT unnest(ARRAY[#{ids.join(',')}]::int[]) AS primary_key) AS imported
63
+ ON imported.primary_key = #{model.table_name}.#{primary_key}
64
+ SQL
65
+ end
62
66
  end
63
67
  end
data/lib/importeur/extractor.rb CHANGED
@@ -21,7 +21,7 @@ module Importeur
21
21
 
22
22
  private
23
23
 
24
- attr_reader :current_dataset_id, :cursor, :cursor_key, :data_source
24
+ attr_reader :cursor, :cursor_key, :data_source
25
25
 
26
26
  def has_new_data?
27
27
  last_known_id = cursor.read(cursor_key)
data/lib/importeur/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Importeur
4
- VERSION = '0.1.0'
4
+ VERSION = '0.1.1'
5
5
  end
metadata CHANGED
@@ -1,73 +1,73 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: importeur
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.1.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Helge Rausch
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2017-10-26 00:00:00.000000000 Z
11
+ date: 2018-01-31 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: bundler
14
+ name: activerecord
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - "~>"
17
+ - - "<"
18
18
  - !ruby/object:Gem::Version
19
- version: '1.15'
19
+ version: '5'
20
20
  type: :development
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - "~>"
24
+ - - "<"
25
25
  - !ruby/object:Gem::Version
26
- version: '1.15'
26
+ version: '5'
27
27
  - !ruby/object:Gem::Dependency
28
- name: rake
28
+ name: appnexusapi
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - "~>"
31
+ - - ">="
32
32
  - !ruby/object:Gem::Version
33
- version: '10.0'
33
+ version: '0'
34
34
  type: :development
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - "~>"
38
+ - - ">="
39
39
  - !ruby/object:Gem::Version
40
- version: '10.0'
40
+ version: '0'
41
41
  - !ruby/object:Gem::Dependency
42
- name: rspec
42
+ name: bucket_cake
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - "~>"
45
+ - - ">="
46
46
  - !ruby/object:Gem::Version
47
- version: '3.0'
47
+ version: '0'
48
48
  type: :development
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - "~>"
52
+ - - ">="
53
53
  - !ruby/object:Gem::Version
54
- version: '3.0'
54
+ version: '0'
55
55
  - !ruby/object:Gem::Dependency
56
- name: activerecord
56
+ name: bundler
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - "<"
59
+ - - "~>"
60
60
  - !ruby/object:Gem::Version
61
- version: '5'
61
+ version: '1.15'
62
62
  type: :development
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - "<"
66
+ - - "~>"
67
67
  - !ruby/object:Gem::Version
68
- version: '5'
68
+ version: '1.15'
69
69
  - !ruby/object:Gem::Dependency
70
- name: pry
70
+ name: database_cleaner
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
73
  - - ">="
@@ -81,7 +81,7 @@ dependencies:
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
83
  - !ruby/object:Gem::Dependency
84
- name: vcr
84
+ name: dotenv
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
87
  - - ">="
@@ -95,21 +95,21 @@ dependencies:
95
95
  - !ruby/object:Gem::Version
96
96
  version: '0'
97
97
  - !ruby/object:Gem::Dependency
98
- name: webmock
98
+ name: pg
99
99
  requirement: !ruby/object:Gem::Requirement
100
100
  requirements:
101
- - - ">="
101
+ - - "<"
102
102
  - !ruby/object:Gem::Version
103
- version: '0'
103
+ version: '1'
104
104
  type: :development
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
- - - ">="
108
+ - - "<"
109
109
  - !ruby/object:Gem::Version
110
- version: '0'
110
+ version: '1'
111
111
  - !ruby/object:Gem::Dependency
112
- name: bucket_cake
112
+ name: pry
113
113
  requirement: !ruby/object:Gem::Requirement
114
114
  requirements:
115
115
  - - ">="
@@ -123,21 +123,21 @@ dependencies:
123
123
  - !ruby/object:Gem::Version
124
124
  version: '0'
125
125
  - !ruby/object:Gem::Dependency
126
- name: pg
126
+ name: rake
127
127
  requirement: !ruby/object:Gem::Requirement
128
128
  requirements:
129
- - - ">="
129
+ - - "~>"
130
130
  - !ruby/object:Gem::Version
131
- version: '0'
131
+ version: '10.0'
132
132
  type: :development
133
133
  prerelease: false
134
134
  version_requirements: !ruby/object:Gem::Requirement
135
135
  requirements:
136
- - - ">="
136
+ - - "~>"
137
137
  - !ruby/object:Gem::Version
138
- version: '0'
138
+ version: '10.0'
139
139
  - !ruby/object:Gem::Dependency
140
- name: database_cleaner
140
+ name: rocketfuel_api
141
141
  requirement: !ruby/object:Gem::Requirement
142
142
  requirements:
143
143
  - - ">="
@@ -151,21 +151,21 @@ dependencies:
151
151
  - !ruby/object:Gem::Version
152
152
  version: '0'
153
153
  - !ruby/object:Gem::Dependency
154
- name: appnexusapi
154
+ name: rspec
155
155
  requirement: !ruby/object:Gem::Requirement
156
156
  requirements:
157
- - - ">="
157
+ - - "~>"
158
158
  - !ruby/object:Gem::Version
159
- version: '0'
159
+ version: '3.0'
160
160
  type: :development
161
161
  prerelease: false
162
162
  version_requirements: !ruby/object:Gem::Requirement
163
163
  requirements:
164
- - - ">="
164
+ - - "~>"
165
165
  - !ruby/object:Gem::Version
166
- version: '0'
166
+ version: '3.0'
167
167
  - !ruby/object:Gem::Dependency
168
- name: rocketfuel_api
168
+ name: vcr
169
169
  requirement: !ruby/object:Gem::Requirement
170
170
  requirements:
171
171
  - - ">="
@@ -179,7 +179,7 @@ dependencies:
179
179
  - !ruby/object:Gem::Version
180
180
  version: '0'
181
181
  - !ruby/object:Gem::Dependency
182
- name: dotenv
182
+ name: webmock
183
183
  requirement: !ruby/object:Gem::Requirement
184
184
  requirements:
185
185
  - - ">="
@@ -240,7 +240,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
240
240
  version: '0'
241
241
  requirements: []
242
242
  rubyforge_project:
243
- rubygems_version: 2.6.13
243
+ rubygems_version: 2.7.3
244
244
  signing_key:
245
245
  specification_version: 4
246
246
  summary: Universal data importer