kumogata-template 0.0.13 → 0.0.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1d28348b53b71edde659be70ab86b2c2b8058701
-  data.tar.gz: cfd0dc291d3c9c694cf32050a348daafaf7c80ed
+  metadata.gz: 2a50c2750395bd793e1bb9210c05e925f6632d4a
+  data.tar.gz: d6faff601811cb365049b0880e731de5f9bb6360
 SHA512:
-  metadata.gz: 6f2f46aa131ffa23e6f6d4f04361f4ba498f55a4f27a2ae52e61c7b9c6c10be948476eeb734211af48e3729a6a5cbbee7046c37b24631d6a54afad8e7364cced
-  data.tar.gz: e0612c919a8c68bf074d19181045ca4f693f8a6d70dd9f59d4a898ba3d7bc6886d5c47dc5863e511053f02562da1f0fe09e5604f11c046ef8d382dd7a3593d52
+  metadata.gz: ab98967ab2b353665b672bb35c3eea3edcf12d0670650603784eb80d4ff97cafcc65ae9a54351b42ba23e2f66e2feb7b7c669abbb0d8797f2788cbc9b27d7882
+  data.tar.gz: a069fe45f07e2dc691f60881ced851ec96b3922acf67648dc66cbe27440f47b7017baf3319ceef17756185665c80b91e8663c87c8dede9d72577fc086617cff6

data/Gemfile.lock

@@ -8,13 +8,13 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-sdk (2.6.39)
-      aws-sdk-resources (= 2.6.39)
-    aws-sdk-core (2.6.39)
+    aws-sdk (2.6.44)
+      aws-sdk-resources (= 2.6.44)
+    aws-sdk-core (2.6.44)
       aws-sigv4 (~> 1.0)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.6.39)
-      aws-sdk-core (= 2.6.39)
+    aws-sdk-resources (2.6.44)
+      aws-sdk-core (= 2.6.44)
     aws-sdk-v1 (1.66.0)
       json (~> 1.4)
       nokogiri (>= 1.4.4)
@@ -30,7 +30,7 @@ GEM
     hashie (3.4.6)
     highline (1.7.8)
     jmespath (1.3.1)
-    json (1.8.3)
+    json (1.8.5)
     json5 (0.0.1)
     kumogata (0.5.10)
       aws-sdk-v1
@@ -50,16 +50,16 @@ GEM
     libv8 (3.16.14.17)
     mini_portile2 (2.1.0)
     minitest (5.10.1)
-    net-ssh (3.2.0)
-    nokogiri (1.6.8.1)
+    net-ssh (4.0.1)
+    nokogiri (1.7.0.1)
       mini_portile2 (~> 2.1.0)
     rake (11.3.0)
     ref (2.0.0)
     retryable (2.0.4)
     term-ansicolor (1.4.0)
       tins (~> 1.0)
-    therubyracer (0.12.2)
-      libv8 (~> 3.16.14.0)
+    therubyracer (0.12.3)
+      libv8 (~> 3.16.14.15)
       ref
     tins (1.13.0)
     uuidtools (2.1.5)
@@ -74,4 +74,4 @@ DEPENDENCIES
   rake (~> 11.1)
 
 BUNDLED WITH
-   1.13.6
+   1.13.7

data/lib/kumogata/template/emr.rb

@@ -4,6 +4,44 @@
 require 'kumogata/template/helper'
 
 
+def _emr_to_configurations_default_hadoop_spark(max_age: 14)
+  _emr_to_configurations_default_hadoop | _emr_to_configurations_default_spark(max_age: max_age)
+end
+
+def _emr_to_configurations_default_hadoop()
+  [
+   {
+     classification: "hadoop-env",
+     properties: {},
+     configurations: [
+                      classification: "export",
+                      properties: { JAVA_HOME: "/usr/java/default" },
+                     ],
+   }
+  ]
+end
+
+def _emr_to_configurations_default_spark(max_age: 14)
+  [
+   {
+     classification: "spark-env",
+     properties: {},
+     configurations: [
+                      classification: "export",
+                      properties: { JAVA_HOME: "/usr/java/default" },
+                     ],
+   },
+   {
+     classification: "spark-defaults",
+     properties: {
+       "spark.history.fs.cleaner.enabled"  => "true",
+       "spark.history.fs.cleaner.interval" => "1d",
+       "spark.history.fs.cleaner.maxAge"  => "#{max_age}d",
+     }
+   }
+  ]
+end
+
 def _emr_applications(args)
   applications = args[:applications] || []
 
@@ -174,7 +212,7 @@ def _emr_step_properties(args)
 
   array = []
   properties.each do |property|
-    _{
+    array << _{
       Key property[:key]
       Value property[:value]
     }

data/lib/kumogata/template/version.rb

@@ -1 +1 @@
-KUMOGATA_TEMPLATE_VERSION = '0.0.13'
+KUMOGATA_TEMPLATE_VERSION = '0.0.14'

data/test/emr_test.rb

@@ -2,6 +2,54 @@ require 'abstract_unit'
 require 'kumogata/template/emr'
 
 class EmrTest < Minitest::Test
+  def test_emr_to_configurations_default_hadoop_spark
+    template = <<-EOS
+Test _emr_to_configurations_default_hadoop_spark(max_age: 1)
+    EOS
+    act_template = run_client_as_json(template)
+    exp_template = <<-EOS
+{
+  "Test": [
+    {
+      "classification": "hadoop-env",
+      "properties": {
+      },
+      "configurations": [
+        {
+          "classification": "export",
+          "properties": {
+            "JAVA_HOME": "/usr/java/default"
+          }
+        }
+      ]
+    },
+    {
+      "classification": "spark-env",
+      "properties": {
+      },
+      "configurations": [
+        {
+          "classification": "export",
+          "properties": {
+            "JAVA_HOME": "/usr/java/default"
+          }
+        }
+      ]
+    },
+    {
+      "classification": "spark-defaults",
+      "properties": {
+        "spark.history.fs.cleaner.enabled": "true",
+        "spark.history.fs.cleaner.interval": "1d",
+        "spark.history.fs.cleaner.maxAge": "1d"
+      }
+    }
+  ]
+}
+    EOS
+    assert_equal exp_template.chomp, act_template
+  end
+
   def test_emr_applications
     template = <<-EOS
 Test _emr_applications(applications: [ { name: "test", version: "test" } ])
@@ -179,9 +227,9 @@ Test _emr_instance_group({ name: "test" })
     assert_equal exp_template.chomp, act_template
   end
 
-  def _emr_hadoop_jar_config
+  def test_emr_hadoop_jar_step_config
     template = <<-EOS
-Test _emr_hadopo_jar_config(jar: "test")
+Test _emr_hadoop_jar_step_config(jar: "test")
     EOS
     act_template = run_client_as_json(template)
     exp_template = <<-EOS
@@ -194,9 +242,9 @@ Test _emr_hadopo_jar_config(jar: "test")
     assert_equal exp_template.chomp, act_template
   end
 
-  def _emr_step_properties
+  def test_emr_step_properties
     template = <<-EOS
-Test _emr_step_properties([ key: "test", value: "test" ])
+Test _emr_step_properties(step_properties: [ { key: "test", value: "test" } ])
     EOS
     act_template = run_client_as_json(template)
     exp_template = <<-EOS
@@ -204,7 +252,7 @@ Test _emr_step_properties([ key: "test", value: "test" ])
   "Test": [
     {
       "Key": "test",
-      "Valuue": "test"
+      "Value": "test"
     }
   ]
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kumogata-template
 version: !ruby/object:Gem::Version
-  version: 0.0.13
+  version: 0.0.14
 platform: ruby
 authors:
 - Naoya Nakazawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-20 00:00:00.000000000 Z
+date: 2017-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk