knife-hadoop 0.0.8 → 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9fa81ed05953792988f58df504fae11f92736765
+  data.tar.gz: 84f278679b4b3c4fbc4b6251a4223f3c9bbb4e25
+SHA512:
+  metadata.gz: 80aefed486cf8dd1d28cb41d75d3cefa4ccf80fbecb5aa4f7f877b14f005b41e9b4b692596266ffb3b64c660de6a4d147e4567fa25fa6e30d75e2cd2065590d3
+  data.tar.gz: 876517ff2a7093e9b22ec39739957d1a414e4cc7a9d0304c05b2a2d24e6e26907c3202fcfc034a8c6a0989bf25b56fe008a02fdf97c90ac37fc8fa82b6376773

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    knife-hadoop (0.0.7)
-      chef
+    knife-hadoop (0.0.8)
+      chef (> 10.24.0)
       debugger
       pg
       rest-client
@@ -28,7 +28,7 @@ GEM
       rest-client (>= 1.0.4, < 1.7.0)
       yajl-ruby (~> 1.1)
     columnize (0.3.6)
-    debugger (1.3.3)
+    debugger (1.4.0)
       columnize (>= 0.3.1)
       debugger-linecache (~> 1.1.1)
       debugger-ruby_core_source (~> 1.2.0)
@@ -46,7 +46,7 @@ GEM
     mixlib-config (1.1.2)
     mixlib-log (1.4.1)
     mixlib-shellout (1.1.0)
-    net-ssh (2.6.5)
+    net-ssh (2.6.6)
     net-ssh-gateway (1.2.0)
       net-ssh (>= 2.6.5)
     net-ssh-multi (1.1)
@@ -63,7 +63,7 @@ GEM
     pg (0.14.1)
     rest-client (1.6.7)
       mime-types (>= 1.16)
-    sequel (3.44.0)
+    sequel (3.45.0)
     sqlite3 (1.3.7)
     systemu (2.5.2)
     webhdfs (0.5.1)

data/README.md

@@ -3,6 +3,15 @@ Knife Hadoop
 
 This is a Chef Knife plugin for Hadoop. This plugin gives knife the ability to provision, list, and manage Hadoop for Operators. 
 
+Version 0.1.0
+Chef 11.x
+
+Version 0.0.9
+
+Added PostgreSQL port option
+General clean up
+
+
 Version 0.0.8
 
 Bug Fixes.
@@ -13,7 +22,7 @@ HDFS APIs (currently supported) using the ruby webhdfs gem: https://github.com/k
 https://github.com/murraju/webhdfs
 	
 	a. List Directories and Files
-	b. Snapshot metadata information to a database (SQLlite default, PostgreSQL). Useful for reporting and audits
+	b. Snapshot metadata information to a database (PostgreSQL or Sqlite). Useful for reporting and audits
 	c. Create Directories and Files
 	d. Update Files
 	e. Read Files
@@ -26,8 +35,9 @@ MapReduce APIs supported using the awesome work done by huahin: https://github.c
 
 Issues:
 
-1. The WebHDFS gem has bugs on net-http for create/delete
-2. Not all methods are exposed
+1. The WebHDFS gem has bugs on net-http for create/delete.
+2. Not all methods are exposed.
+3. HDFS usage still in development.
 
 
 
@@ -57,6 +67,7 @@ In order to communicate with Hadoop and other APIs, you will have to set paramet
 	knife[:db_username] 		= "dbusername"
 	knife[:db_password] 		= "dbpassword"
 	knife[:db_host] 			= "dbhost"
+	knife[:db_host] 			= "port"
 	knife[:db] 					= "db"
 
 If your knife.rb file will be checked into a SCM system (ie readable by others) you may want to read the values from environment variables:
@@ -70,6 +81,7 @@ If your knife.rb file will be checked into a SCM system (ie readable by others)
 	knife[:db_username] 		= "#{ENV['DB_USERNAME']}"
 	knife[:db_password] 		= "#{ENV['DB_PASSWORD']}"
 	knife[:db_host] 			= "#{ENV['DB_HOST']}"
+	knife[:db_host] 			= "#{ENV['DB_PORT']}"
 	knife[:db] 					= "#{ENV['DB']}"
 
 

data/knife-hadoop.gemspec

@@ -25,5 +25,5 @@ Gem::Specification.new do |s|
   s.add_dependency "sequel"
   s.add_dependency "debugger"
   s.add_dependency "rest-client"
-  s.add_dependency "chef"
+  s.add_dependency "chef", "> 11.0.0"
 end

data/lib/chef/knife/hadoop_base.rb

@@ -87,6 +87,12 @@ class Chef
             :long => "--db-host DBHOST",
             :description => "PostgreSQL DB Host",
             :proc => Proc.new { |key| Chef::Config[:knife][:db_host] = key }
+
+            option :db_port,
+              :short => "-F DBPORT",
+              :long => "--db-port DBPORT",
+              :description => "PostgreSQL DB Port",
+              :proc => Proc.new { |key| Chef::Config[:knife][:db_port] = key }
           
         end
       end
@@ -109,14 +115,14 @@ class Chef
         Chef::Log.debug("db_username: #{Chef::Config[:knife][:db_username]}")
         Chef::Log.debug("db_password: #{Chef::Config[:knife][:db_password]}")
         Chef::Log.debug("db_host: #{Chef::Config[:knife][:db_host]}")
+        Chef::Log.debug("db_port: #{Chef::Config[:knife][:db_port]}")
         db_type = "#{Chef::Config[:knife][:db_type]}".downcase
         case db_type
         when 'postgres'
           @db_connection ||= begin
-            db_connection = Sequel.connect("postgres://#{Chef::Config[:knife][:db_username]}"+':'+
-                                           "#{Chef::Config[:knife][:db_password]}"+'@'+
-                                           "#{Chef::Config[:knife][:db_host]}"+'/'+ 
-                                           "#{Chef::Config[:knife][:db]}")
+            db_connection = Sequel.postgres("#{Chef::Config[:knife][:db]}", :user=>"#{Chef::Config[:knife][:db_username]}", 
+                                             :password => "#{Chef::Config[:knife][:db_password]}", :host => "#{Chef::Config[:knife][:db_host]}", 
+                                             :port => "#{Chef::Config[:knife][:db_port]}", :max_connections => 10)
           end
         when 'sqlite'
           @db_connection ||= begin

data/lib/chef/knife/hadoop_hdfs_usage.rb

@@ -0,0 +1,134 @@
+# Author:: Murali Raju (<murali.raju@appliv.com>)
+# Copyright:: Copyright (c) 2012 Murali Raju.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/knife/hadoop_base'
+
+class Chef
+  class Knife
+    class HadoopHdfsUsage < Knife
+
+      include Knife::HadoopBase
+
+      deps do
+        require 'readline'
+        require 'chef/json_compat'
+        require 'chef/knife/bootstrap'
+        require 'net/ssh'
+        Chef::Knife::Bootstrap.load_deps
+      end
+
+      banner "knife hadoop hdfs usage (options)"
+      
+      option :type,
+        :short => "-T TYPE",
+        :long => "--type TYPE",
+        :description => "The type <summary,detail,report>",
+        :proc => Proc.new { |f| Chef::Config[:knife][:type] = f }
+
+      option :dir,
+        :short => "-D DIRECTORY",
+        :long => "--hdfs-directory DIRECTORY",
+        :description => "The HDFS Directory to use",
+        :proc => Proc.new { |f| Chef::Config[:knife][:dir] = f }
+
+      option :ssh_user,
+        :short => "-U SSHUSER",
+        :long => "--ssh-user SSHUSER",
+        :description => "The SSH User",
+        :proc => Proc.new { |f| Chef::Config[:knife][:ssh_user] = f }
+
+        option :ssh_password,
+          :short => "-P SSHPASSWORD",
+          :long => "--ssh-password SSHPASSWORD",
+          :description => "The SSH User Password",
+          :proc => Proc.new { |f| Chef::Config[:knife][:ssh_password] = f }
+
+      def run
+        $stdout.sync = true
+        
+        hdfs_usage_summary_list = [
+          ui.color('Configured Capacity',           :bold),
+          ui.color('Present Capacity',              :bold),
+          ui.color('DFS Remaining',                 :bold),
+          ui.color('DFS Used',                      :bold),
+          ui.color('DFS Used%',                     :bold),
+          ui.color('Under replicated blocks',       :bold),
+          ui.color('Blocks with corrupt replicas',  :bold),
+          ui.color('Missing blocks',                :bold),
+          ui.color('Datanodes available',           :bold)
+        ]
+
+        hdfs_usage_node_list = [
+          ui.color('Data Node',                     :bold),
+          ui.color('Decommission Status',           :bold),
+          ui.color('Configured Capacity',           :bold),
+          ui.color('DFS Used',                      :bold),
+          ui.color('Non DFS Used',                  :bold),
+          ui.color('DFS Remaining',                 :bold),
+          ui.color('DFS Used%',                     :bold),
+          ui.color('DFS Remaining%',                :bold),
+          ui.color('Last contact',                  :bold)
+        ]        
+        
+        type = "#{Chef::Config[:knife][:type]}".downcase
+        case type
+        when 'summary'
+          Net::SSH.start( "#{Chef::Config[:knife][:namenode_host]}", 
+                          "#{Chef::Config[:knife][:ssh_user]}", :password => "#{Chef::Config[:knife][:ssh_password]}" ) do|ssh|
+            result = ssh.exec!('hadoop dfsadmin -report')
+            hdfs_usage_summary_list << result.match(/Configured Capacity: \d+(.*?)/).to_s.split(':')[1].gsub(/\s+/, "")
+            hdfs_usage_summary_list << result.match(/Present Capacity: \d+(.*?)/).to_s.split(':')[1].gsub(/\s+/, "")
+            hdfs_usage_summary_list << result.match(/DFS Remaining: \d+(.*?)/).to_s.split(':')[1].gsub(/\s+/, "")
+            hdfs_usage_summary_list << result.match(/DFS Used: \d+(.*?)/).to_s.split(':')[1].gsub(/\s+/, "")
+            hdfs_usage_summary_list << result.match(/DFS Used%: \d+(.*?).*/).to_s.split(':')[1].gsub(/\s+/, "")
+            hdfs_usage_summary_list << result.match(/Under replicated blocks: \d+(.*?)/).to_s.split(':')[1].gsub(/\s+/, "")
+            hdfs_usage_summary_list << result.match(/Blocks with corrupt replicas: \d+(.*?)/).to_s.split(':')[1].gsub(/\s+/, "")
+            hdfs_usage_summary_list << result.match(/Missing blocks: \d+(.*?)/).to_s.split(':')[1].gsub(/\s+/, "")
+            hdfs_usage_summary_list << result.match(/Datanodes available: \d+(.*?)/).to_s.split(':')[1].gsub(/\s+/, "")
+          end
+          puts ui.list(hdfs_usage_summary_list, :uneven_columns_across, 9)
+        when 'detail'
+          Net::SSH.start( "#{Chef::Config[:knife][:namenode_host]}", 
+                          "#{Chef::Config[:knife][:ssh_user]}", :password => "#{Chef::Config[:knife][:ssh_password]}" ) do|ssh|
+            result = ssh.exec!('hadoop dfsadmin -report')
+            hdfs_usage_node_list << result.match(/Name: \d+(.*?).*/).to_s.split(':')[1].gsub(/\s+/, "")
+            hdfs_usage_node_list << result.match(/Decommission Status : \w+(.*?).*/).to_s.split(':')[1].gsub(/\s+/, "")
+            hdfs_usage_node_list << result.match(/Configured Capacity: \d+(.*?)/).to_s.split(':')[1].gsub(/\s+/, "")
+            hdfs_usage_node_list << result.match(/DFS Used: \d+(.*?)/).to_s.split(':')[1].gsub(/\s+/, "")
+            hdfs_usage_node_list << result.match(/Non DFS Used: \d+(.*?)/).to_s.split(':')[1].gsub(/\s+/, "")
+            hdfs_usage_node_list << result.match(/DFS Remaining: \d+(.*?)/).to_s.split(':')[1].gsub(/\s+/, "")
+            hdfs_usage_node_list << result.match(/DFS Used%: \d+(.*?).*/).to_s.split(':')[1].gsub(/\s+/, "")
+            hdfs_usage_node_list << result.match(/DFS Remaining%: \d+(.*?).*/).to_s.split(':')[1].gsub(/\s+/, "")
+            hdfs_usage_node_list << result.match(/Last contact: \w+(.*?) .*/).to_s.split(':')[1] 
+          end
+          puts ui.list(hdfs_usage_node_list, :uneven_columns_across, 9)
+        when 'report'
+          Net::SSH.start( "#{Chef::Config[:knife][:namenode_host]}", 
+                          "#{Chef::Config[:knife][:ssh_user]}", :password => "#{Chef::Config[:knife][:ssh_password]}" ) do|ssh|
+            result = ssh.exec!('hadoop dfsadmin -report')
+            file = "hdfs_usage_report_created_on_#{Time.now}.txt"
+            File.open("/tmp/#{file}", 'w') do |f|
+              f.write(result)
+              f.close
+            end
+            hdfs_connection.create("#{Chef::Config[:knife][:dir]}/#{file}", result)
+          end
+        end  
+      end
+    end
+  end
+end

data/lib/chef/knife/hadoop_mapred_job_list.rb

@@ -24,6 +24,7 @@ class Chef
 
       deps do
         require 'readline'
+        require 'rest-client'
         require 'chef/json_compat'
         require 'chef/knife/bootstrap'
         Chef::Knife::Bootstrap.load_deps

data/lib/knife-hadoop/version.rb

@@ -1,6 +1,6 @@
 module Knife
   module Hadoop
-    VERSION = "0.0.8"
+    VERSION = "0.1.0"
     MAJOR, MINOR, TINY = VERSION.split('.')
   end
 end

metadata

@@ -1,128 +1,113 @@
 --- !ruby/object:Gem::Specification
 name: knife-hadoop
 version: !ruby/object:Gem::Version
-  version: 0.0.8
-  prerelease: 
+  version: 0.1.0
 platform: ruby
 authors:
 - Murali Raju
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-26 00:00:00.000000000 Z
+date: 2013-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: webhdfs
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: debugger
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rest-client
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: chef
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>'
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 11.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>'
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 11.0.0
 description: Hadoop Chef Knife Plugin
 email:
 - murraju@appliv.com
@@ -144,32 +129,32 @@ files:
 - lib/chef/knife/hadoop_hdfs_list.rb
 - lib/chef/knife/hadoop_hdfs_snapshot.rb
 - lib/chef/knife/hadoop_hdfs_update.rb
+- lib/chef/knife/hadoop_hdfs_usage.rb
 - lib/chef/knife/hadoop_mapred_job_kill.rb
 - lib/chef/knife/hadoop_mapred_job_list.rb
 - lib/chef/knife/hadoop_setup.rb
 - lib/knife-hadoop/version.rb
 homepage: https://github.com/murraju/knife-hadoop
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Hadoop Chef Knife Plugin
 test_files: []