fog-hadoop 0.1.1

data/lib/fog/hadoop.rb

@@ -0,0 +1,24 @@
+require 'fog/core'
+require 'fog/json'
+require 'fog/hadoop/utils/request'
+
+module Fog
+  module Compute
+    autoload :Hadoop, File.expand_path('../hadoop/compute', __FILE__)
+  end
+
+  module Hadoop
+    extend Fog::Provider
+
+
+    # Miscs
+    ## Startup Script
+    #autoload :Script,      File.expand_path('../sakuracloud/script', __FILE__)
+
+    service(:compute, 'Compute')
+
+    def self.yarn_endpoint
+      "/ws/v1/cluster"
+    end
+  end
+end

data/lib/fog/hadoop/compute.rb

@@ -0,0 +1,35 @@
+module Fog
+  module Compute
+    class Hadoop < Fog::Service
+      requires     :hadoop_compute_api_url
+
+      recognizes   :compute_api_url
+
+
+      request_path 'fog/hadoop/requests/compute'
+      request      :get_info
+      request      :get_app_stats
+      request      :get_app_stats_detail
+      request      :get_metrics
+      request      :get_node
+      request      :list_nodes
+
+      class Real
+        include Fog::Hadoop::Utils::Request
+
+        def initialize(options = {})
+
+          @compute_api_url = options[:hadoop_compute_api_url] || 'https://localhost:8088/'
+          Fog.credentials[:@compute_api_url]        = options[:hadoop_compute_api_url]
+          @connection = Fog::Core::Connection.new(@compute_api_url)
+        end
+      end
+
+      class Mock
+        def initialize(options = {})
+          @compute_api_url = options[:hadoop_compute_api_url] || 'https://localhost:8088/'
+        end
+      end
+    end #SakuraCloud
+  end #Compute
+end

data/lib/fog/hadoop/requests/compute/get_app_stats.rb

@@ -0,0 +1,40 @@
+module Fog
+  module Compute
+    class Hadoop
+      class Real
+        def get_app_stats
+          request(
+              :expects => [200, 203],
+              :method => 'GET',
+              :path => "#{Fog::Hadoop.yarn_endpoint}/appstatistics"
+          ).body["appStatInfo"]["statItem"]
+        end
+      end
+      class Mock
+        def get_app_stats
+          response = Excon::Response.new
+          response.status = 200
+          response.body =
+              [
+                {
+                  "state" => "accepted",
+                  "type"  => "mapreduce",
+                  "count" => 4
+                },
+                {
+                  "state" => "running",
+                  "type"  =>  "mapreduce",
+                  "count" => 1
+                },
+                {
+                  "state" => "finished",
+                  "type"  => "mapreduce",
+                  "count" => 7
+                }
+              ]
+          response
+        end
+      end
+    end
+  end
+end

data/lib/fog/hadoop/requests/compute/get_app_stats_detail.rb

@@ -0,0 +1,30 @@
+module Fog
+  module Compute
+    class Hadoop
+      class Real
+        def get_app_stats_detail(options = {})
+          request(
+              :expects => [200, 203],
+              :method => 'GET',
+              :path => "#{Fog::Hadoop.yarn_endpoint}/appstatistics?#{URI.encode_www_form(options)}"
+          ).body["appStatInfo"]["statItem"]
+        end
+      end
+      class Mock
+        def get_app_stats_detail(options = {})
+          response = Excon::Response.new
+          response.status = 200
+          response.body =
+              [
+                {
+                  "state" => "accepted",
+                  "type"  => "mapreduce",
+                  "count" => 4
+                }
+              ]
+          response
+        end
+      end
+    end
+  end
+end

data/lib/fog/hadoop/requests/compute/get_info.rb

@@ -0,0 +1,37 @@
+module Fog
+  module Compute
+    class Hadoop
+      class Real
+        def get_info
+          request(
+              :expects => [200, 203],
+              :method => 'GET',
+              :path => "#{Fog::Hadoop.yarn_endpoint}/info"
+          ).body["clusterInfo"]
+        end
+      end
+      class Mock
+        def get_info
+          response = Excon::Response.new
+          response.status = 200
+          response.body =
+              {
+                "id" => 1324053971963,
+                "startedOn" => 1324053971963,
+                "state" => "STARTED",
+                "resourceManagerVersion" => "0.23.1-SNAPSHOT",
+                "resourceManagerBuildVersion" => "0.23.1-SNAPSHOT from 1214049 by user1"\
+                                                 " source checksum 050cd664439d931c8743a6428fd6a693",
+                "resourceManagerVersionBuiltOn" => "Tue Dec 13 22:12:48 CST 2011",
+                "hadoopVersion" => "0.23.1-SNAPSHOT",
+                "hadoopBuildVersion" => "0.23.1-SNAPSHOT from 1214049 by user1"\
+                                        " source checksum 11458df3bb77342dca5f917198fad328",
+                "hadoopVersionBuiltOn" => "Tue Dec 13 22:12:26 CST 2011"
+
+              }
+          response
+        end
+      end
+    end
+  end
+end

data/lib/fog/hadoop/requests/compute/get_metrics.rb

@@ -0,0 +1,49 @@
+module Fog
+  module Compute
+    class Hadoop
+      class Real
+        def get_metrics
+          request(
+              :expects => 200,
+              :method => 'GET',
+              :path => "#{Fog::Hadoop.yarn_endpoint}/metrics"
+          ).body["clusterMetrics"]
+        end
+      end
+      class Mock
+        def get_metrics
+          response = Excon::Response.new
+          response.status = 200
+          response.body =
+                  {
+                      "appsSubmitted"         => 0,
+                      "appsCompleted"         => 0,
+                      "appsPending"           => 0,
+                      "appsRunning"           => 0,
+                      "appsFailed"            => 0,
+                      "appsKilled"            => 0,
+                      "reservedMB"            => 0,
+                      "availableMB"           => 17408,
+                      "allocatedMB"           => 0,
+                      "reservedVirtualCores"  => 0,
+                      "availableVirtualCores" => 7,
+                      "allocatedVirtualCores" => 1,
+                      "containersAllocated"   => 0,
+                      "containersReserved"    => 0,
+                      "containersPending"     => 0,
+                      "totalMB"               => 17408,
+                      "totalVirtualCores"     => 8,
+                      "totalNodes"            => 1,
+                      "lostNodes"             => 0,
+                      "unhealthyNodes"        => 0,
+                      "decommissionedNodes"   => 0,
+                      "rebootedNodes"         => 0,
+                      "activeNodes"           => 1
+                  }
+
+          response
+        end
+      end
+    end
+  end
+end

data/lib/fog/hadoop/requests/compute/get_node.rb

@@ -0,0 +1,38 @@
+module Fog
+  module Compute
+    class Hadoop
+      class Real
+        def get_node(node_ref)
+          request(
+              :expects => [200, 203],
+              :method => 'GET',
+              :path => "#{Fog::Hadoop.yarn_endpoint}/nodes/#{node_ref}"
+          ).body["node"]
+        end
+      end
+      class Mock
+        def get_node(node_ref)
+          response = Excon::Response.new
+          response.status = 200
+          response.body =
+              {
+                  "rack"                  => "\/default-rack",
+                  "state"                 => "NEW",
+                  "id"                    => "h2:1235",
+                  "nodeHostName"          => "h2",
+                  "nodeHTTPAddress"       => "h2:2",
+                  "healthStatus"          => "Healthy",
+                  "lastHealthUpdate"      => 1324056895432,
+                  "healthReport"          => "Healthy",
+                  "numContainers"         => 0,
+                  "usedMemoryMB"          => 0,
+                  "availMemoryMB"         => 5120,
+                  "usedVirtualCores"      => 0,
+                  "availableVirtualCores" => 8
+              }
+          response
+        end
+      end
+    end
+  end
+end

data/lib/fog/hadoop/requests/compute/list_nodes.rb

@@ -0,0 +1,55 @@
+module Fog
+  module Compute
+    class Hadoop
+      class Real
+        def list_nodes
+          request(
+              :expects => 200,
+              :method => 'GET',
+              :path => "#{Fog::Hadoop.yarn_endpoint}/nodes"
+          ).body["nodes"]["node"]
+        end
+      end
+      class Mock
+        def list_nodes
+          response = Excon::Response.new
+          response.status = 200
+          response.body =
+              [
+                  {
+                      "rack"                  => "\/default-rack",
+                      "state"                 => "NEW",
+                      "id"                    => "h2:1235",
+                      "nodeHostName"          => "h2",
+                      "nodeHTTPAddress"       => "h2:2",
+                      "healthStatus"          => "Healthy",
+                      "lastHealthUpdate"      => 1324056895432,
+                      "healthReport"          => "Healthy",
+                      "numContainers"         => 0,
+                      "usedMemoryMB"          => 0,
+                      "availMemoryMB"         => 8192,
+                      "usedVirtualCores"      => 0,
+                      "availableVirtualCores" => 8
+                  },
+                  {
+                      "rack"                  => "\/default-rack",
+                      "state"                 => "NEW",
+                      "id"                    => "h1:1234",
+                      "nodeHostName"          => "h1",
+                      "nodeHTTPAddress"       => "h1:2",
+                      "healthStatus"          => "Healthy",
+                      "lastHealthUpdate"      => 1324056895092,
+                      "healthReport"          => "Healthy",
+                      "numContainers"         => 0,
+                      "usedMemoryMB"          => 0,
+                      "availMemoryMB"         => 8192,
+                      "usedVirtualCores"      => 0,
+                      "availableVirtualCores" => 8
+                  }
+              ]
+          response
+        end
+      end
+    end
+  end
+end

data/lib/fog/hadoop/script.rb

@@ -0,0 +1,22 @@
+module Fog
+  module Hadoop
+    class Script < Fog::Service
+
+      class Real
+        include Fog::Hadoop::Utils::Request
+
+        def initialize(options = {})
+
+
+          @connection = Fog::Core::Connection.new(@service_api_url)
+        end
+      end
+
+      class Mock
+        def initialize(options={})
+          @service_api_url        = options[:service_api_url]
+        end
+      end
+    end
+  end
+end

data/lib/fog/hadoop/utils/request.rb

@@ -0,0 +1,27 @@
+require 'json'
+
+module Fog
+  module Hadoop
+    module Utils
+      module Request
+        def request(params)
+          response = parse @connection.request(params)
+
+          response
+            ## SakuraCloud API returns Japanese message.
+            #  This wrapper decodes and show message to be human readble.
+        rescue Excon::Errors::HTTPStatusError => e
+          Fog::Logger.warning ::JSON.parse(e.response.body)['error_msg']
+          raise e
+        end
+
+        private
+        def parse(response)
+          return response if response.body.empty?
+          response.body = Fog::JSON.decode(response.body)
+          response
+        end
+      end
+    end
+  end
+end

data/lib/fog/hadoop/version.rb

@@ -0,0 +1,5 @@
+module Fog
+  module Hadoop
+    VERSION = "0.1.1"
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,6 @@
+# require 'coveralls'
+# Coveralls.wear!
+
+require 'minitest/autorun'
+require 'fog/core'
+require 'fog/openstack'

data/supported.md

@@ -0,0 +1,19 @@
+# Supported Hadoop Projects
+
+## Supported
+
+| Project          | Fog Type      | API Version(s) | Compliance | Notes |
+|------------------|---------------|----------------|------------|-------|
+| Yarn             | Compute       | v1             | TBD        |       |
+| Hdfs             | Volume        | v1             | TBD        |       |
+
+
+
+## YARN
+
+| Service          | GET           |Notes                                |
+|------------------|---------------|-------------------------------------|
+| Yarn             | Info          |/ws/v1/cluster/info                  |
+|                  | AppStats      |/ws/v1/cluster/appstatistics         |
+|                  | Metrics       |/ws/v1/cluster/metrics               |
+|                  | Nodes         |/ws/v1/cluster/nodes                 |

data/tests/hadoop/requests/compute/app_stats_tests.rb

@@ -0,0 +1,44 @@
+Shindo.tests('Fog::Compute[:hadoop] | app stats request', ['hadoop', 'compute']) do
+
+    @app_stats_format = {
+        "state" => String,
+        "type"  => String,
+        "count" => Integer
+    }
+
+  tests('success') do
+    tests('#get_app_stats') do
+      stats = hadoop_compute_service.get_app_stats
+      test 'returns a Hash' do
+        stats.body.is_a? Array
+      end
+      if Fog.mock?
+        tests('Info').formats(@app_stats_format, false) do
+          stats.body.first
+        end
+        tests('Info').formats(@app_stats_format, false) do
+          stats.body.last
+        end
+      else
+        returns(200) { stats.status }
+        returns(true) { stats.body.is_a? Hash }
+      end
+    end
+
+    tests('#get_app_stats_detail') do
+      stats_detail = hadoop_compute_service.get_app_stats_detail(:states => "accepted")
+      test 'returns a Hash' do
+        stats_detail.body.is_a? Array
+      end
+      if Fog.mock?
+        tests('Info').formats(@app_stats_format, false) do
+          stats_detail.body.first
+        end
+      else
+        returns(200) { stats_detail.status }
+        returns(true) { stats_detail.body.is_a? Hash }
+      end
+    end
+  end
+end
+