sql2avro 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/interface/mysql.rb +2 -5
- data/lib/sql2avro.rb +2 -2
- metadata +5 -5
data/lib/interface/mysql.rb
CHANGED
|
@@ -47,7 +47,7 @@ class MySql < DbInterface
|
|
|
47
47
|
end
|
|
48
48
|
end
|
|
49
49
|
|
|
50
|
-
def data(table, min_id, max_id)
|
|
50
|
+
def data(table, min_id, max_id, &block)
|
|
51
51
|
columns = nil
|
|
52
52
|
rows = []
|
|
53
53
|
|
|
@@ -100,11 +100,8 @@ class MySql < DbInterface
|
|
|
100
100
|
end
|
|
101
101
|
end
|
|
102
102
|
|
|
103
|
-
|
|
103
|
+
block.call(row)
|
|
104
104
|
end
|
|
105
|
-
|
|
106
|
-
# TODO: stream this data out rather than return all in one batch.
|
|
107
|
-
rows
|
|
108
105
|
end
|
|
109
106
|
|
|
110
107
|
def sql_schema(table)
|
data/lib/sql2avro.rb
CHANGED
|
@@ -52,7 +52,7 @@ module Sql2Avro
|
|
|
52
52
|
begin
|
|
53
53
|
json_file = "#{filename}.json"
|
|
54
54
|
File.open(json_file, 'w') do |f|
|
|
55
|
-
interface.data(table, min_id, max_id_this_batch)
|
|
55
|
+
interface.data(table, min_id, max_id_this_batch) do |datum|
|
|
56
56
|
Yajl::Encoder.encode(datum, f)
|
|
57
57
|
f.write "\n"
|
|
58
58
|
end
|
|
@@ -61,7 +61,7 @@ module Sql2Avro
|
|
|
61
61
|
cmd = "java -jar #{AVRO_TOOLS_PATH} fromjson --codec snappy --schema '#{schema}' #{json_file} > #{filename}"
|
|
62
62
|
`#{cmd}`
|
|
63
63
|
|
|
64
|
-
|
|
64
|
+
#`rm #{json_file}`
|
|
65
65
|
rescue Exception => e
|
|
66
66
|
retval[:error] = "#{e}\n\n#{e.backtrace}"
|
|
67
67
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: sql2avro
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
prerelease:
|
|
6
6
|
platform: ruby
|
|
7
7
|
authors:
|
|
@@ -13,7 +13,7 @@ date: 2013-06-21 00:00:00.000000000Z
|
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: yajl-ruby
|
|
16
|
-
requirement: &
|
|
16
|
+
requirement: &70363993723780 !ruby/object:Gem::Requirement
|
|
17
17
|
none: false
|
|
18
18
|
requirements:
|
|
19
19
|
- - ! '>='
|
|
@@ -21,7 +21,7 @@ dependencies:
|
|
|
21
21
|
version: '0'
|
|
22
22
|
type: :runtime
|
|
23
23
|
prerelease: false
|
|
24
|
-
version_requirements: *
|
|
24
|
+
version_requirements: *70363993723780
|
|
25
25
|
description: sql2avro extracts data from a specified SQL database table and transforms
|
|
26
26
|
it into an Avro file with a schema based on the database table's schema. The intended
|
|
27
27
|
use case is to incrementally load data out of an SQL database and into HDFS for
|
|
@@ -52,7 +52,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
52
52
|
version: '0'
|
|
53
53
|
segments:
|
|
54
54
|
- 0
|
|
55
|
-
hash:
|
|
55
|
+
hash: 1878707749575910576
|
|
56
56
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
57
57
|
none: false
|
|
58
58
|
requirements:
|
|
@@ -61,7 +61,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
61
61
|
version: '0'
|
|
62
62
|
segments:
|
|
63
63
|
- 0
|
|
64
|
-
hash:
|
|
64
|
+
hash: 1878707749575910576
|
|
65
65
|
requirements: []
|
|
66
66
|
rubyforge_project:
|
|
67
67
|
rubygems_version: 1.8.10
|