trinamo 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +44 -11
- data/lib/trinamo/converter/base_converter.rb +1 -1
- data/lib/trinamo/converter/dynamodb_converter.rb +1 -10
- data/lib/trinamo/converter/option_converter.rb +15 -0
- data/lib/trinamo/converter.rb +20 -2
- data/lib/trinamo/version.rb +1 -1
- data/trinamo.gemspec +1 -0
- metadata +17 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e454cf16de7a293b800e5a0b5df17f25f41ec906
|
4
|
+
data.tar.gz: c2d1f3752aa75cc7e49f5c467f5a23dc7d6f5eeb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1bba7a9132864946c464ed01b026cdd578ffa33f8f171fb5602a222000b0e30b945942eb6fde05732f0d22dab7ddf16e45337e99c23ab49e3416b155ddd5c0e2
|
7
|
+
data.tar.gz: d7010d9882d28dee138a9e9984090b04f97ef1553b517bb34bdb210a057c28bdd6ccacd43066da2877b31b4acc9dae5149683730dc20c50061baa486bf116240
|
data/README.md
CHANGED
@@ -1,7 +1,9 @@
|
|
1
1
|
# Trinamo
|
2
2
|
|
3
|
-
|
4
|
-
|
3
|
+
[](https://travis-ci.org/cignoir/trinamo)
|
4
|
+
[](https://coveralls.io/github/cignoir/trinamo?branch=master)
|
5
|
+
|
6
|
+
Trinamo generates HiveQL using YAML to mount tables of DynamoDB, S3 and local HDFS.
|
5
7
|
|
6
8
|
```
|
7
9
|
Notice:
|
@@ -26,16 +28,15 @@ Or install it yourself as:
|
|
26
28
|
|
27
29
|
## Usage
|
28
30
|
|
29
|
-
|
30
|
-
|
31
|
+
## Table Definition
|
32
|
+
### Generate a template for DDL
|
31
33
|
* RUN:
|
32
34
|
```ruby
|
33
|
-
Trinamo::Converter.
|
35
|
+
Trinamo::Converter.generate_ddl_template(out_file_path = 'ddl.yml')
|
34
36
|
```
|
35
37
|
|
36
38
|
* OUTPUT:
|
37
39
|
```yaml
|
38
|
-
dynamo_read_percent: 0.75
|
39
40
|
tables:
|
40
41
|
- name: comments
|
41
42
|
s3_location: s3://path/to/s3/table/location
|
@@ -64,20 +65,52 @@ tables:
|
|
64
65
|
type: string
|
65
66
|
```
|
66
67
|
|
67
|
-
###
|
68
|
+
### Generate a template for hive options
|
69
|
+
* RUN:
|
70
|
+
```ruby
|
71
|
+
Trinamo::Converter.generate_options_template(out_file_path = 'ddl.yml')
|
72
|
+
```
|
73
|
+
|
74
|
+
* OUTPUT:
|
75
|
+
```yaml
|
76
|
+
options:
|
77
|
+
dynamodb.throughput.read.percent: 0.5
|
78
|
+
hive.exec.compress.output: true
|
79
|
+
io.seqfile.compression.type: BLOCK
|
80
|
+
mapred.output.compression.codec: com.hadoop.compression.lzo.LzoCodec
|
81
|
+
|
82
|
+
```
|
83
|
+
|
84
|
+
Then, modify table-definitions and hive-settings as you like.
|
68
85
|
|
86
|
+
## Create DDLs in HiveQL
|
87
|
+
### For Options
|
69
88
|
* RUN:
|
70
89
|
```ruby
|
71
|
-
Trinamo::Converter.load('ddl.yml', :
|
90
|
+
Trinamo::Converter.load('ddl.yml', :option).convert
|
91
|
+
```
|
92
|
+
or
|
93
|
+
```ruby
|
94
|
+
Trinamo::Converter.load_options('options.yml').convert
|
72
95
|
```
|
73
96
|
|
74
97
|
* OUTPUT:
|
75
98
|
```hql
|
76
|
-
SET dynamodb.throughput.read.percent = 0.
|
99
|
+
SET dynamodb.throughput.read.percent = 0.5;
|
77
100
|
SET hive.exec.compress.output=true;
|
78
101
|
SET io.seqfile.compression.type=BLOCK;
|
79
102
|
SET mapred.output.compression.codec = com.hadoop.compression.lzo.LzoCodec;
|
103
|
+
```
|
80
104
|
|
105
|
+
### For DynamoDB
|
106
|
+
|
107
|
+
* RUN:
|
108
|
+
```ruby
|
109
|
+
Trinamo::Converter.load('ddl.yml', :dynamodb).convert
|
110
|
+
```
|
111
|
+
|
112
|
+
* OUTPUT:
|
113
|
+
```hql
|
81
114
|
-- comments_ddb
|
82
115
|
CREATE EXTERNAL TABLE comments_ddb (
|
83
116
|
user_id BIGINT,comment_id BIGINT,title STRING,content STRING,rate DOUBLE
|
@@ -99,7 +132,7 @@ TBLPROPERTIES (
|
|
99
132
|
);
|
100
133
|
```
|
101
134
|
|
102
|
-
###
|
135
|
+
### For S3
|
103
136
|
* RUN:
|
104
137
|
```ruby
|
105
138
|
Trinamo::Converter.load('ddl.yml', :s3).convert
|
@@ -115,7 +148,7 @@ ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n'
|
|
115
148
|
LOCATION 's3://path/to/s3/table/location';
|
116
149
|
```
|
117
150
|
|
118
|
-
###
|
151
|
+
### For HDFS
|
119
152
|
* RUN:
|
120
153
|
```ruby
|
121
154
|
Trinamo::Converter.load('ddl.yml', :hdfs).convert
|
@@ -3,15 +3,6 @@ require_relative './base_converter'
|
|
3
3
|
module Trinamo
|
4
4
|
class DynamodbConverter < BaseConverter
|
5
5
|
def convert
|
6
|
-
read_percent = @ddl[:dynamo_read_percent] ? @ddl[:dynamo_read_percent] : 0.5
|
7
|
-
|
8
|
-
ddl_header = <<-DDL.unindent
|
9
|
-
SET dynamodb.throughput.read.percent = #{read_percent};
|
10
|
-
SET hive.exec.compress.output=true;
|
11
|
-
SET io.seqfile.compression.type=BLOCK;
|
12
|
-
SET mapred.output.compression.codec = com.hadoop.compression.lzo.LzoCodec;
|
13
|
-
DDL
|
14
|
-
|
15
6
|
ddl_body = @ddl[:tables].map do |h|
|
16
7
|
fields = ([h[:hash_key]] + [h[:range_key]] + [h[:attributes]]).flatten.compact
|
17
8
|
<<-DDL.unindent
|
@@ -27,7 +18,7 @@ module Trinamo
|
|
27
18
|
DDL
|
28
19
|
end
|
29
20
|
|
30
|
-
|
21
|
+
ddl_body.join("\n")
|
31
22
|
end
|
32
23
|
end
|
33
24
|
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require_relative './base_converter'
|
2
|
+
|
3
|
+
module Trinamo
|
4
|
+
class OptionConverter < BaseConverter
|
5
|
+
def convert
|
6
|
+
options = @ddl[:options] ? @ddl[:options] : {}
|
7
|
+
|
8
|
+
queries = options.keys.map do |key|
|
9
|
+
options[key] ? "SET #{key} = #{options[key]};" : nil
|
10
|
+
end
|
11
|
+
|
12
|
+
queries.compact.join("\n")
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
data/lib/trinamo/converter.rb
CHANGED
@@ -2,6 +2,7 @@ require 'unindent'
|
|
2
2
|
require_relative './converter/dynamodb_converter'
|
3
3
|
require_relative './converter/hdfs_converter'
|
4
4
|
require_relative './converter/s3_converter'
|
5
|
+
require_relative './converter/option_converter'
|
5
6
|
|
6
7
|
module Trinamo
|
7
8
|
class Converter
|
@@ -11,13 +12,17 @@ module Trinamo
|
|
11
12
|
when :hdfs then HdfsConverter.new(ddl_yaml_path)
|
12
13
|
when :s3 then S3Converter.new(ddl_yaml_path)
|
13
14
|
when :dynamodb then DynamodbConverter.new(ddl_yaml_path)
|
15
|
+
when :option || :options then load_options(ddl_yaml_path)
|
14
16
|
else raise "[ERROR] Unknown format: #{format}" unless [:dynamodb, :hdfs, :s3].include(format)
|
15
17
|
end
|
16
18
|
end
|
17
19
|
|
18
|
-
def
|
20
|
+
def load_options(options_yaml_path)
|
21
|
+
OptionConverter.new(options_yaml_path)
|
22
|
+
end
|
23
|
+
|
24
|
+
def generate_ddl_template(out_file_path = nil)
|
19
25
|
template = <<-TEMPLATE.unindent
|
20
|
-
dynamo_read_percent: 0.75
|
21
26
|
tables:
|
22
27
|
- name: comments
|
23
28
|
s3_location: s3://path/to/s3/table/location
|
@@ -49,6 +54,19 @@ module Trinamo
|
|
49
54
|
File.binwrite(out_file_path, template) if out_file_path
|
50
55
|
template
|
51
56
|
end
|
57
|
+
|
58
|
+
def generate_options_template(out_file_path = nil)
|
59
|
+
template = <<-TEMPLATE.unindent
|
60
|
+
options:
|
61
|
+
dynamodb.throughput.read.percent: 0.5
|
62
|
+
hive.exec.compress.output: true
|
63
|
+
io.seqfile.compression.type: BLOCK
|
64
|
+
mapred.output.compression.codec: com.hadoop.compression.lzo.LzoCodec
|
65
|
+
TEMPLATE
|
66
|
+
|
67
|
+
File.binwrite(out_file_path, template) if out_file_path
|
68
|
+
template
|
69
|
+
end
|
52
70
|
end
|
53
71
|
end
|
54
72
|
end
|
data/lib/trinamo/version.rb
CHANGED
data/trinamo.gemspec
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: trinamo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- cignoir
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-06-
|
11
|
+
date: 2016-06-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -80,6 +80,20 @@ dependencies:
|
|
80
80
|
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '3.0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: coveralls
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
83
97
|
description: ''
|
84
98
|
email:
|
85
99
|
- cignoir@gmail.com
|
@@ -101,6 +115,7 @@ files:
|
|
101
115
|
- lib/trinamo/converter/base_converter.rb
|
102
116
|
- lib/trinamo/converter/dynamodb_converter.rb
|
103
117
|
- lib/trinamo/converter/hdfs_converter.rb
|
118
|
+
- lib/trinamo/converter/option_converter.rb
|
104
119
|
- lib/trinamo/converter/s3_converter.rb
|
105
120
|
- lib/trinamo/hive_type.rb
|
106
121
|
- lib/trinamo/version.rb
|