sql2avro 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/sql2avro.rb +17 -21
- metadata +6 -6
data/lib/sql2avro.rb
CHANGED
@@ -49,32 +49,28 @@ module Sql2Avro
|
|
49
49
|
path: File.join(directory, filename)
|
50
50
|
}
|
51
51
|
|
52
|
-
|
53
|
-
|
54
|
-
Encoding.default_internal = nil
|
52
|
+
prev_default_internal = Encoding.default_internal
|
53
|
+
Encoding.default_internal = nil
|
55
54
|
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
end
|
55
|
+
json_file = File.join(directory, "#{filename}.json")
|
56
|
+
File.open(json_file, 'w') do |f|
|
57
|
+
interface.data(table, min_id, max_id_this_batch) do |datum|
|
58
|
+
Yajl::Encoder.encode(datum, f)
|
59
|
+
f.write "\n"
|
62
60
|
end
|
61
|
+
end
|
63
62
|
|
64
|
-
|
63
|
+
Encoding.default_internal = prev_default_internal
|
65
64
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
65
|
+
cmd = "java -jar #{AVRO_TOOLS_PATH} fromjson --codec snappy --schema '#{schema}' #{json_file} > #{File.join(directory, filename)}"
|
66
|
+
`#{cmd}`
|
67
|
+
if !$?.success?
|
68
|
+
raise "Error converting JSON to Avro.\n\nCommand: #{cmd}\nStatus: #{$?}"
|
69
|
+
end
|
71
70
|
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
end
|
76
|
-
rescue Exception => e
|
77
|
-
retval[:error] = "#{e}\n\n#{e.backtrace}"
|
71
|
+
`rm #{json_file}`
|
72
|
+
if !$?.success?
|
73
|
+
raise "Error deleting temporary JSON file #{json_file}"
|
78
74
|
end
|
79
75
|
|
80
76
|
retval
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sql2avro
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-
|
12
|
+
date: 2013-07-30 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: yajl-ruby
|
16
|
-
requirement: &
|
16
|
+
requirement: &70283680850780 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,7 +21,7 @@ dependencies:
|
|
21
21
|
version: '0'
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *70283680850780
|
25
25
|
description: sql2avro extracts data from a specified SQL database table and transforms
|
26
26
|
it into an Avro file with a schema based on the database table's schema. The intended
|
27
27
|
use case is to incrementally load data out of an SQL database and into HDFS for
|
@@ -52,7 +52,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
52
52
|
version: '0'
|
53
53
|
segments:
|
54
54
|
- 0
|
55
|
-
hash:
|
55
|
+
hash: 1417159064959416851
|
56
56
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
57
57
|
none: false
|
58
58
|
requirements:
|
@@ -61,7 +61,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
61
61
|
version: '0'
|
62
62
|
segments:
|
63
63
|
- 0
|
64
|
-
hash:
|
64
|
+
hash: 1417159064959416851
|
65
65
|
requirements: []
|
66
66
|
rubyforge_project:
|
67
67
|
rubygems_version: 1.8.10
|