couchdb_to_sql 2.0.0 → 2.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.rubocop.yml +1 -0
- data/README.md +1 -1
- data/VERSION +1 -1
- data/examples/feed.rb +0 -1
- data/lib/couchdb_to_sql/changes.rb +20 -16
- data/lib/couchdb_to_sql/table_builder.rb +2 -0
- data/test/test_helper.rb +0 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: e8c9e4a2bbd735eb84d80280debaaf5bfe00a937a3e0612c172385242c8e9620
|
4
|
+
data.tar.gz: c6348013a668c3dea0cde87ac4fa8ed5968aba97ee24a631db87bc4a4e4ef37c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 045d9d4017ad0385b37ea36f9dd3af164b50f6e444a973b86b7fa0787f50e23d62fec48785a875752a82bb8a6dcd6152cc9101c8ef9c9599b8cc6c117c01d9fa
|
7
|
+
data.tar.gz: c08676bf07ca8b8e05db22ce2f60759cb1b42523779cb5672b73a8fc6dd46554a7749722e28c0abc3340fb4e343574b366c0aea01e716a4facc38d2723bff95a
|
data/.rubocop.yml
CHANGED
data/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
[![Build Status](https://travis-ci.org/ecraft/couch_tap.svg?branch=master)](https://travis-ci.org/ecraft/couch_tap)
|
1
|
+
[![Build Status](https://travis-ci.org/ecraft/couch_tap.svg?branch=master)](https://travis-ci.org/ecraft/couch_tap) [![Gem Version](https://badge.fury.io/rb/couchdb_to_sql.svg)](https://badge.fury.io/rb/couchdb_to_sql)
|
2
2
|
|
3
3
|
# couchdb_to_sql
|
4
4
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
2.
|
1
|
+
2.1.0
|
data/examples/feed.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module CouchdbToSql
|
4
|
-
class Changes
|
4
|
+
class Changes # rubocop:disable ClassLength
|
5
5
|
COUCHDB_HEARTBEAT = 30
|
6
6
|
INACTIVITY_TIMEOUT = 70
|
7
7
|
RECONNECT_TIMEOUT = 15
|
@@ -129,8 +129,9 @@ module CouchdbToSql
|
|
129
129
|
|
130
130
|
protected
|
131
131
|
|
132
|
-
def perform_request
|
132
|
+
def perform_request # rubocop:disable AbcSize
|
133
133
|
raise 'Internal error: Highest_sequence is expected to be non-nil' unless highest_sequence
|
134
|
+
|
134
135
|
log_info "listening to changes feed from sequence number: #{highest_sequence}"
|
135
136
|
|
136
137
|
url = File.join(source.root.to_s, '_changes')
|
@@ -148,25 +149,28 @@ module CouchdbToSql
|
|
148
149
|
}
|
149
150
|
|
150
151
|
num_rows = 0
|
151
|
-
|
152
152
|
loop do
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
153
|
+
begin
|
154
|
+
# Perform the actual request for chunked content
|
155
|
+
@http.get_content(url, query) do |chunk|
|
156
|
+
rows = chunk.split("\n")
|
157
|
+
rows.each { |row|
|
158
|
+
parsed_row = JSON.parse(row)
|
159
|
+
process_row(parsed_row)
|
160
|
+
|
161
|
+
num_rows += 1
|
162
|
+
log_info "Processed #{num_rows} rows" if (num_rows % 10_000) == 0
|
163
|
+
}
|
164
|
+
end
|
165
|
+
rescue StandardError => e
|
166
|
+
log_error "exception occurred: #{e.message}"
|
167
|
+
log_error "connection ended, attempting to reconnect in #{RECONNECT_TIMEOUT}s..."
|
168
|
+
sleep RECONNECT_TIMEOUT
|
163
169
|
end
|
164
|
-
log_error "connection ended, attempting to reconnect in #{RECONNECT_TIMEOUT}s..."
|
165
|
-
wait RECONNECT_TIMEOUT
|
166
170
|
end
|
167
171
|
rescue HTTPClient::TimeoutError, HTTPClient::BadResponseError => e
|
168
172
|
log_error "connection failed: #{e.message}, attempting to reconnect in #{RECONNECT_TIMEOUT}s..."
|
169
|
-
|
173
|
+
sleep RECONNECT_TIMEOUT
|
170
174
|
retry
|
171
175
|
end
|
172
176
|
|
@@ -83,9 +83,11 @@ module CouchdbToSql
|
|
83
83
|
# Take the document and try to automatically set the fields from the columns
|
84
84
|
def set_attributes_from_data
|
85
85
|
return unless data.is_a?(Hash) || data.is_a?(CouchRest::Document)
|
86
|
+
|
86
87
|
data.each do |k, v|
|
87
88
|
k = k.to_sym
|
88
89
|
next if %i[_id _rev].include?(k)
|
90
|
+
|
89
91
|
set_attribute(k, v) if schema.column_names.include?(k)
|
90
92
|
end
|
91
93
|
end
|
data/test/test_helper.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: couchdb_to_sql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sam Lown
|
@@ -11,7 +11,7 @@ authors:
|
|
11
11
|
autorequire:
|
12
12
|
bindir: exe
|
13
13
|
cert_chain: []
|
14
|
-
date:
|
14
|
+
date: 2019-06-19 00:00:00.000000000 Z
|
15
15
|
dependencies:
|
16
16
|
- !ruby/object:Gem::Dependency
|
17
17
|
name: activesupport
|
@@ -219,7 +219,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
219
219
|
version: '0'
|
220
220
|
requirements: []
|
221
221
|
rubyforge_project:
|
222
|
-
rubygems_version: 2.
|
222
|
+
rubygems_version: 2.7.8
|
223
223
|
signing_key:
|
224
224
|
specification_version: 4
|
225
225
|
summary: Listen to a CouchDB changes feed and create rows in a relational database
|