trinamo 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: aaf548285a2c17c8ef8913b70e42058c5dd7d1d2
4
- data.tar.gz: 223310522802f3482caa2b46556c068a4f1cd9a4
3
+ metadata.gz: e454cf16de7a293b800e5a0b5df17f25f41ec906
4
+ data.tar.gz: c2d1f3752aa75cc7e49f5c467f5a23dc7d6f5eeb
5
5
  SHA512:
6
- metadata.gz: 6ef96ddbe9510141e62835a003e9cbfa03ea518e59fd348ac257bd48b578ef1a0f7e77b6947d50f4cb77e0cd79b40b51e00fc6690fcf2d351cf2bc99ff6412af
7
- data.tar.gz: 58e45dcfe9dd86704894282028a5a6f5666f758f0d76052cd52e3e628acfa73c7d90fd55909e51b7941727452a7391b70d78dcf6c2b9b0be7145926d12a02f9c
6
+ metadata.gz: 1bba7a9132864946c464ed01b026cdd578ffa33f8f171fb5602a222000b0e30b945942eb6fde05732f0d22dab7ddf16e45337e99c23ab49e3416b155ddd5c0e2
7
+ data.tar.gz: d7010d9882d28dee138a9e9984090b04f97ef1553b517bb34bdb210a057c28bdd6ccacd43066da2877b31b4acc9dae5149683730dc20c50061baa486bf116240
data/README.md CHANGED
@@ -1,7 +1,9 @@
1
1
  # Trinamo
2
2
 
3
- Trinamo generates DDL for Hive from YAML
4
- to mount tables of DynamoDB, S3 and local HDFS.
3
+ [![Build Status](https://travis-ci.org/cignoir/trinamo.svg?branch=master)](https://travis-ci.org/cignoir/trinamo)
4
+ [![Coverage Status](https://coveralls.io/repos/github/cignoir/trinamo/badge.svg?branch=master)](https://coveralls.io/github/cignoir/trinamo?branch=master)
5
+
6
+ Trinamo generates HiveQL using YAML to mount tables of DynamoDB, S3 and local HDFS.
5
7
 
6
8
  ```
7
9
  Notice:
@@ -26,16 +28,15 @@ Or install it yourself as:
26
28
 
27
29
  ## Usage
28
30
 
29
- ### Create a DDL template
30
-
31
+ ## Table Definition
32
+ ### Generate a template for DDL
31
33
  * RUN:
32
34
  ```ruby
33
- Trinamo::Converter.generate_template(out_file_path = 'ddl.yml')
35
+ Trinamo::Converter.generate_ddl_template(out_file_path = 'ddl.yml')
34
36
  ```
35
37
 
36
38
  * OUTPUT:
37
39
  ```yaml
38
- dynamo_read_percent: 0.75
39
40
  tables:
40
41
  - name: comments
41
42
  s3_location: s3://path/to/s3/table/location
@@ -64,20 +65,52 @@ tables:
64
65
  type: string
65
66
  ```
66
67
 
67
- ### Create a mapper for DynamoDB
68
+ ### Generate a template for hive options
69
+ * RUN:
70
+ ```ruby
71
+ Trinamo::Converter.generate_options_template(out_file_path = 'ddl.yml')
72
+ ```
73
+
74
+ * OUTPUT:
75
+ ```yaml
76
+ options:
77
+ dynamodb.throughput.read.percent: 0.5
78
+ hive.exec.compress.output: true
79
+ io.seqfile.compression.type: BLOCK
80
+ mapred.output.compression.codec: com.hadoop.compression.lzo.LzoCodec
81
+
82
+ ```
83
+
84
+ Then, modify table-definitions and hive-settings as you like.
68
85
 
86
+ ## Create DDLs in HiveQL
87
+ ### For Options
69
88
  * RUN:
70
89
  ```ruby
71
- Trinamo::Converter.load('ddl.yml', :dynamodb).convert
90
+ Trinamo::Converter.load('ddl.yml', :option).convert
91
+ ```
92
+ or
93
+ ```ruby
94
+ Trinamo::Converter.load_options('options.yml').convert
72
95
  ```
73
96
 
74
97
  * OUTPUT:
75
98
  ```hql
76
- SET dynamodb.throughput.read.percent = 0.75;
99
+ SET dynamodb.throughput.read.percent = 0.5;
77
100
  SET hive.exec.compress.output=true;
78
101
  SET io.seqfile.compression.type=BLOCK;
79
102
  SET mapred.output.compression.codec = com.hadoop.compression.lzo.LzoCodec;
103
+ ```
80
104
 
105
+ ### For DynamoDB
106
+
107
+ * RUN:
108
+ ```ruby
109
+ Trinamo::Converter.load('ddl.yml', :dynamodb).convert
110
+ ```
111
+
112
+ * OUTPUT:
113
+ ```hql
81
114
  -- comments_ddb
82
115
  CREATE EXTERNAL TABLE comments_ddb (
83
116
  user_id BIGINT,comment_id BIGINT,title STRING,content STRING,rate DOUBLE
@@ -99,7 +132,7 @@ TBLPROPERTIES (
99
132
  );
100
133
  ```
101
134
 
102
- ### Create a mapper for S3
135
+ ### For S3
103
136
  * RUN:
104
137
  ```ruby
105
138
  Trinamo::Converter.load('ddl.yml', :s3).convert
@@ -115,7 +148,7 @@ ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n'
115
148
  LOCATION 's3://path/to/s3/table/location';
116
149
  ```
117
150
 
118
- ### Create a mapper for HDFS local
151
+ ### For HDFS
119
152
  * RUN:
120
153
  ```ruby
121
154
  Trinamo::Converter.load('ddl.yml', :hdfs).convert
@@ -7,7 +7,7 @@ module Trinamo
7
7
 
8
8
  def initialize(ddl_yaml_path)
9
9
  @ddl_yaml_path = ddl_yaml_path
10
- @ddl = YAML.load_file(ddl_yaml_path).deep_symbolize_keys
10
+ @ddl = ddl_yaml_path ? YAML.load_file(ddl_yaml_path).deep_symbolize_keys : nil
11
11
  end
12
12
  end
13
13
  end
@@ -3,15 +3,6 @@ require_relative './base_converter'
3
3
  module Trinamo
4
4
  class DynamodbConverter < BaseConverter
5
5
  def convert
6
- read_percent = @ddl[:dynamo_read_percent] ? @ddl[:dynamo_read_percent] : 0.5
7
-
8
- ddl_header = <<-DDL.unindent
9
- SET dynamodb.throughput.read.percent = #{read_percent};
10
- SET hive.exec.compress.output=true;
11
- SET io.seqfile.compression.type=BLOCK;
12
- SET mapred.output.compression.codec = com.hadoop.compression.lzo.LzoCodec;
13
- DDL
14
-
15
6
  ddl_body = @ddl[:tables].map do |h|
16
7
  fields = ([h[:hash_key]] + [h[:range_key]] + [h[:attributes]]).flatten.compact
17
8
  <<-DDL.unindent
@@ -27,7 +18,7 @@ module Trinamo
27
18
  DDL
28
19
  end
29
20
 
30
- ([ddl_header] + ddl_body).join("\n")
21
+ ddl_body.join("\n")
31
22
  end
32
23
  end
33
24
  end
@@ -0,0 +1,15 @@
1
+ require_relative './base_converter'
2
+
3
+ module Trinamo
4
+ class OptionConverter < BaseConverter
5
+ def convert
6
+ options = @ddl[:options] ? @ddl[:options] : {}
7
+
8
+ queries = options.keys.map do |key|
9
+ options[key] ? "SET #{key} = #{options[key]};" : nil
10
+ end
11
+
12
+ queries.compact.join("\n")
13
+ end
14
+ end
15
+ end
@@ -2,6 +2,7 @@ require 'unindent'
2
2
  require_relative './converter/dynamodb_converter'
3
3
  require_relative './converter/hdfs_converter'
4
4
  require_relative './converter/s3_converter'
5
+ require_relative './converter/option_converter'
5
6
 
6
7
  module Trinamo
7
8
  class Converter
@@ -11,13 +12,17 @@ module Trinamo
11
12
  when :hdfs then HdfsConverter.new(ddl_yaml_path)
12
13
  when :s3 then S3Converter.new(ddl_yaml_path)
13
14
  when :dynamodb then DynamodbConverter.new(ddl_yaml_path)
15
+ when :option || :options then load_options(ddl_yaml_path)
14
16
  else raise "[ERROR] Unknown format: #{format}" unless [:dynamodb, :hdfs, :s3].include(format)
15
17
  end
16
18
  end
17
19
 
18
- def generate_template(out_file_path = nil)
20
+ def load_options(options_yaml_path)
21
+ OptionConverter.new(options_yaml_path)
22
+ end
23
+
24
+ def generate_ddl_template(out_file_path = nil)
19
25
  template = <<-TEMPLATE.unindent
20
- dynamo_read_percent: 0.75
21
26
  tables:
22
27
  - name: comments
23
28
  s3_location: s3://path/to/s3/table/location
@@ -49,6 +54,19 @@ module Trinamo
49
54
  File.binwrite(out_file_path, template) if out_file_path
50
55
  template
51
56
  end
57
+
58
+ def generate_options_template(out_file_path = nil)
59
+ template = <<-TEMPLATE.unindent
60
+ options:
61
+ dynamodb.throughput.read.percent: 0.5
62
+ hive.exec.compress.output: true
63
+ io.seqfile.compression.type: BLOCK
64
+ mapred.output.compression.codec: com.hadoop.compression.lzo.LzoCodec
65
+ TEMPLATE
66
+
67
+ File.binwrite(out_file_path, template) if out_file_path
68
+ template
69
+ end
52
70
  end
53
71
  end
54
72
  end
@@ -1,3 +1,3 @@
1
1
  module Trinamo
2
- VERSION = "0.2.0"
2
+ VERSION = "0.3.0"
3
3
  end
data/trinamo.gemspec CHANGED
@@ -24,4 +24,5 @@ Gem::Specification.new do |spec|
24
24
  spec.add_development_dependency "bundler", "~> 1.11"
25
25
  spec.add_development_dependency "rake", "~> 10.0"
26
26
  spec.add_development_dependency "rspec", "~> 3.0"
27
+ spec.add_development_dependency(%q<coveralls>, [">= 0"])
27
28
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: trinamo
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - cignoir
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2016-06-09 00:00:00.000000000 Z
11
+ date: 2016-06-10 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -80,6 +80,20 @@ dependencies:
80
80
  - - "~>"
81
81
  - !ruby/object:Gem::Version
82
82
  version: '3.0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: coveralls
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
83
97
  description: ''
84
98
  email:
85
99
  - cignoir@gmail.com
@@ -101,6 +115,7 @@ files:
101
115
  - lib/trinamo/converter/base_converter.rb
102
116
  - lib/trinamo/converter/dynamodb_converter.rb
103
117
  - lib/trinamo/converter/hdfs_converter.rb
118
+ - lib/trinamo/converter/option_converter.rb
104
119
  - lib/trinamo/converter/s3_converter.rb
105
120
  - lib/trinamo/hive_type.rb
106
121
  - lib/trinamo/version.rb