trinamo 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: aaf548285a2c17c8ef8913b70e42058c5dd7d1d2
-  data.tar.gz: 223310522802f3482caa2b46556c068a4f1cd9a4
+  metadata.gz: e454cf16de7a293b800e5a0b5df17f25f41ec906
+  data.tar.gz: c2d1f3752aa75cc7e49f5c467f5a23dc7d6f5eeb
 SHA512:
-  metadata.gz: 6ef96ddbe9510141e62835a003e9cbfa03ea518e59fd348ac257bd48b578ef1a0f7e77b6947d50f4cb77e0cd79b40b51e00fc6690fcf2d351cf2bc99ff6412af
-  data.tar.gz: 58e45dcfe9dd86704894282028a5a6f5666f758f0d76052cd52e3e628acfa73c7d90fd55909e51b7941727452a7391b70d78dcf6c2b9b0be7145926d12a02f9c
+  metadata.gz: 1bba7a9132864946c464ed01b026cdd578ffa33f8f171fb5602a222000b0e30b945942eb6fde05732f0d22dab7ddf16e45337e99c23ab49e3416b155ddd5c0e2
+  data.tar.gz: d7010d9882d28dee138a9e9984090b04f97ef1553b517bb34bdb210a057c28bdd6ccacd43066da2877b31b4acc9dae5149683730dc20c50061baa486bf116240

data/README.md

@@ -1,7 +1,9 @@
 # Trinamo
 
-Trinamo generates DDL for Hive from YAML
-to mount tables of DynamoDB, S3 and local HDFS.
+[![Build Status](https://travis-ci.org/cignoir/trinamo.svg?branch=master)](https://travis-ci.org/cignoir/trinamo)
+[![Coverage Status](https://coveralls.io/repos/github/cignoir/trinamo/badge.svg?branch=master)](https://coveralls.io/github/cignoir/trinamo?branch=master)
+
+Trinamo generates HiveQL using YAML to mount tables of DynamoDB, S3 and local HDFS.
 
 ```
 Notice:
@@ -26,16 +28,15 @@ Or install it yourself as:
 
 ## Usage
 
-### Create a DDL template
-
+## Table Definition
+### Generate a template for DDL
 * RUN:
 ```ruby
-Trinamo::Converter.generate_template(out_file_path = 'ddl.yml')
+Trinamo::Converter.generate_ddl_template(out_file_path = 'ddl.yml')
 ```
 
 * OUTPUT:
 ```yaml
-dynamo_read_percent: 0.75
 tables:
   - name: comments
     s3_location: s3://path/to/s3/table/location
@@ -64,20 +65,52 @@ tables:
         type: string
 ```
 
-### Create a mapper for DynamoDB
+### Generate a template for hive options
+* RUN:
+```ruby
+Trinamo::Converter.generate_options_template(out_file_path = 'ddl.yml')
+```
+
+* OUTPUT:
+```yaml
+options:
+  dynamodb.throughput.read.percent: 0.5
+  hive.exec.compress.output: true
+  io.seqfile.compression.type: BLOCK
+  mapred.output.compression.codec: com.hadoop.compression.lzo.LzoCodec
+
+```
+
+Then, modify table-definitions and hive-settings as you like.
 
+## Create DDLs in HiveQL
+### For Options
 * RUN:
 ```ruby
-Trinamo::Converter.load('ddl.yml', :dynamodb).convert
+Trinamo::Converter.load('ddl.yml', :option).convert
+```
+or
+```ruby
+Trinamo::Converter.load_options('options.yml').convert
 ```
 
 * OUTPUT:
 ```hql
-SET dynamodb.throughput.read.percent = 0.75;
+SET dynamodb.throughput.read.percent = 0.5;
 SET hive.exec.compress.output=true;
 SET io.seqfile.compression.type=BLOCK;
 SET mapred.output.compression.codec = com.hadoop.compression.lzo.LzoCodec;
+```
 
+### For DynamoDB
+
+* RUN:
+```ruby
+Trinamo::Converter.load('ddl.yml', :dynamodb).convert
+```
+
+* OUTPUT:
+```hql
 -- comments_ddb
 CREATE EXTERNAL TABLE comments_ddb (
   user_id BIGINT,comment_id BIGINT,title STRING,content STRING,rate DOUBLE
@@ -99,7 +132,7 @@ TBLPROPERTIES (
 );
 ```
 
-### Create a mapper for S3
+### For S3
 * RUN:
 ```ruby
 Trinamo::Converter.load('ddl.yml', :s3).convert
@@ -115,7 +148,7 @@ ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n'
 LOCATION 's3://path/to/s3/table/location';
 ```
 
-### Create a mapper for HDFS local
+### For HDFS
 * RUN:
 ```ruby
 Trinamo::Converter.load('ddl.yml', :hdfs).convert

data/lib/trinamo/converter/base_converter.rb

@@ -7,7 +7,7 @@ module Trinamo
 
     def initialize(ddl_yaml_path)
       @ddl_yaml_path = ddl_yaml_path
-      @ddl = YAML.load_file(ddl_yaml_path).deep_symbolize_keys
+      @ddl = ddl_yaml_path ? YAML.load_file(ddl_yaml_path).deep_symbolize_keys : nil
     end
   end
 end

data/lib/trinamo/converter/dynamodb_converter.rb

@@ -3,15 +3,6 @@ require_relative './base_converter'
 module Trinamo
   class DynamodbConverter < BaseConverter
     def convert
-      read_percent = @ddl[:dynamo_read_percent] ? @ddl[:dynamo_read_percent] : 0.5
-
-      ddl_header = <<-DDL.unindent
-        SET dynamodb.throughput.read.percent = #{read_percent};
-        SET hive.exec.compress.output=true;
-        SET io.seqfile.compression.type=BLOCK;
-        SET mapred.output.compression.codec = com.hadoop.compression.lzo.LzoCodec;
-      DDL
-
       ddl_body = @ddl[:tables].map do |h|
         fields = ([h[:hash_key]] + [h[:range_key]] + [h[:attributes]]).flatten.compact
         <<-DDL.unindent
@@ -27,7 +18,7 @@ module Trinamo
         DDL
       end
 
-      ([ddl_header] + ddl_body).join("\n")
+      ddl_body.join("\n")
     end
   end
 end

data/lib/trinamo/converter/option_converter.rb

@@ -0,0 +1,15 @@
+require_relative './base_converter'
+
+module Trinamo
+  class OptionConverter < BaseConverter
+    def convert
+      options = @ddl[:options] ? @ddl[:options] : {}
+
+      queries = options.keys.map do |key|
+        options[key] ? "SET #{key} = #{options[key]};" : nil
+      end
+
+      queries.compact.join("\n")
+    end
+  end
+end

data/lib/trinamo/converter.rb

@@ -2,6 +2,7 @@ require 'unindent'
 require_relative './converter/dynamodb_converter'
 require_relative './converter/hdfs_converter'
 require_relative './converter/s3_converter'
+require_relative './converter/option_converter'
 
 module Trinamo
   class Converter
@@ -11,13 +12,17 @@ module Trinamo
           when :hdfs then HdfsConverter.new(ddl_yaml_path)
           when :s3 then S3Converter.new(ddl_yaml_path)
           when :dynamodb then DynamodbConverter.new(ddl_yaml_path)
+          when :option || :options then  load_options(ddl_yaml_path)
           else raise "[ERROR] Unknown format: #{format}" unless [:dynamodb, :hdfs, :s3].include(format)
         end
       end
 
-      def generate_template(out_file_path = nil)
+      def load_options(options_yaml_path)
+        OptionConverter.new(options_yaml_path)
+      end
+
+      def generate_ddl_template(out_file_path = nil)
         template = <<-TEMPLATE.unindent
-          dynamo_read_percent: 0.75
           tables:
             - name: comments
               s3_location: s3://path/to/s3/table/location
@@ -49,6 +54,19 @@ module Trinamo
         File.binwrite(out_file_path, template) if out_file_path
         template
       end
+
+      def generate_options_template(out_file_path = nil)
+        template = <<-TEMPLATE.unindent
+          options:
+            dynamodb.throughput.read.percent: 0.5
+            hive.exec.compress.output: true
+            io.seqfile.compression.type: BLOCK
+            mapred.output.compression.codec: com.hadoop.compression.lzo.LzoCodec
+        TEMPLATE
+
+        File.binwrite(out_file_path, template) if out_file_path
+        template
+      end
     end
   end
 end

data/lib/trinamo/version.rb

@@ -1,3 +1,3 @@
 module Trinamo
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/trinamo.gemspec

@@ -24,4 +24,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.11"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency(%q<coveralls>, [">= 0"])
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: trinamo
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - cignoir
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-06-09 00:00:00.000000000 Z
+date: 2016-06-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ''
 email:
 - cignoir@gmail.com
@@ -101,6 +115,7 @@ files:
 - lib/trinamo/converter/base_converter.rb
 - lib/trinamo/converter/dynamodb_converter.rb
 - lib/trinamo/converter/hdfs_converter.rb
+- lib/trinamo/converter/option_converter.rb
 - lib/trinamo/converter/s3_converter.rb
 - lib/trinamo/hive_type.rb
 - lib/trinamo/version.rb