knife-hadoop 0.0.4

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in knife-ucs.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,75 @@
+PATH
+  remote: .
+  specs:
+    knife-hadoop (0.0.1)
+      chef
+      debugger
+      pg
+      rest-client
+      sequel
+      webhdfs
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    chef (11.4.0)
+      erubis
+      highline (>= 1.6.9)
+      json (>= 1.4.4, <= 1.7.7)
+      mixlib-authentication (>= 1.3.0)
+      mixlib-cli (~> 1.3.0)
+      mixlib-config (>= 1.1.2)
+      mixlib-log (>= 1.3.0)
+      mixlib-shellout
+      net-ssh (~> 2.6)
+      net-ssh-multi (~> 1.1.0)
+      ohai (>= 0.6.0)
+      rest-client (>= 1.0.4, < 1.7.0)
+      yajl-ruby (~> 1.1)
+    columnize (0.3.6)
+    debugger (1.3.1)
+      columnize (>= 0.3.1)
+      debugger-linecache (~> 1.1.1)
+      debugger-ruby_core_source (~> 1.1.8)
+    debugger-linecache (1.1.2)
+      debugger-ruby_core_source (>= 1.1.1)
+    debugger-ruby_core_source (1.1.8)
+    erubis (2.7.0)
+    highline (1.6.15)
+    ipaddress (0.8.0)
+    json (1.7.7)
+    mime-types (1.21)
+    mixlib-authentication (1.3.0)
+      mixlib-log
+    mixlib-cli (1.3.0)
+    mixlib-config (1.1.2)
+    mixlib-log (1.4.1)
+    mixlib-shellout (1.1.0)
+    net-ssh (2.6.5)
+    net-ssh-gateway (1.2.0)
+      net-ssh (>= 2.6.5)
+    net-ssh-multi (1.1)
+      net-ssh (>= 2.1.4)
+      net-ssh-gateway (>= 0.99.0)
+    ohai (6.16.0)
+      ipaddress
+      mixlib-cli
+      mixlib-config
+      mixlib-log
+      mixlib-shellout
+      systemu
+      yajl-ruby
+    pg (0.14.1)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+    sequel (3.44.0)
+    systemu (2.5.2)
+    webhdfs (0.5.1)
+    yajl-ruby (1.1.0)
+
+PLATFORMS
+  java
+  ruby
+
+DEPENDENCIES
+  knife-hadoop!

data/LICENSE

@@ -0,0 +1,201 @@
+                              Apache License
+                        Version 2.0, January 2004
+                     http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+   "License" shall mean the terms and conditions for use, reproduction,
+   and distribution as defined by Sections 1 through 9 of this document.
+
+   "Licensor" shall mean the copyright owner or entity authorized by
+   the copyright owner that is granting the License.
+
+   "Legal Entity" shall mean the union of the acting entity and all
+   other entities that control, are controlled by, or are under common
+   control with that entity. For the purposes of this definition,
+   "control" means (i) the power, direct or indirect, to cause the
+   direction or management of such entity, whether by contract or
+   otherwise, or (ii) ownership of fifty percent (50%) or more of the
+   outstanding shares, or (iii) beneficial ownership of such entity.
+
+   "You" (or "Your") shall mean an individual or Legal Entity
+   exercising permissions granted by this License.
+
+   "Source" form shall mean the preferred form for making modifications,
+   including but not limited to software source code, documentation
+   source, and configuration files.
+
+   "Object" form shall mean any form resulting from mechanical
+   transformation or translation of a Source form, including but
+   not limited to compiled object code, generated documentation,
+   and conversions to other media types.
+
+   "Work" shall mean the work of authorship, whether in Source or
+   Object form, made available under the License, as indicated by a
+   copyright notice that is included in or attached to the work
+   (an example is provided in the Appendix below).
+
+   "Derivative Works" shall mean any work, whether in Source or Object
+   form, that is based on (or derived from) the Work and for which the
+   editorial revisions, annotations, elaborations, or other modifications
+   represent, as a whole, an original work of authorship. For the purposes
+   of this License, Derivative Works shall not include works that remain
+   separable from, or merely link (or bind by name) to the interfaces of,
+   the Work and Derivative Works thereof.
+
+   "Contribution" shall mean any work of authorship, including
+   the original version of the Work and any modifications or additions
+   to that Work or Derivative Works thereof, that is intentionally
+   submitted to Licensor for inclusion in the Work by the copyright owner
+   or by an individual or Legal Entity authorized to submit on behalf of
+   the copyright owner. For the purposes of this definition, "submitted"
+   means any form of electronic, verbal, or written communication sent
+   to the Licensor or its representatives, including but not limited to
+   communication on electronic mailing lists, source code control systems,
+   and issue tracking systems that are managed by, or on behalf of, the
+   Licensor for the purpose of discussing and improving the Work, but
+   excluding communication that is conspicuously marked or otherwise
+   designated in writing by the copyright owner as "Not a Contribution."
+
+   "Contributor" shall mean Licensor and any individual or Legal Entity
+   on behalf of whom a Contribution has been received by Licensor and
+   subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of
+   this License, each Contributor hereby grants to You a perpetual,
+   worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+   copyright license to reproduce, prepare Derivative Works of,
+   publicly display, publicly perform, sublicense, and distribute the
+   Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of
+   this License, each Contributor hereby grants to You a perpetual,
+   worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+   (except as stated in this section) patent license to make, have made,
+   use, offer to sell, sell, import, and otherwise transfer the Work,
+   where such license applies only to those patent claims licensable
+   by such Contributor that are necessarily infringed by their
+   Contribution(s) alone or by combination of their Contribution(s)
+   with the Work to which such Contribution(s) was submitted. If You
+   institute patent litigation against any entity (including a
+   cross-claim or counterclaim in a lawsuit) alleging that the Work
+   or a Contribution incorporated within the Work constitutes direct
+   or contributory patent infringement, then any patent licenses
+   granted to You under this License for that Work shall terminate
+   as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the
+   Work or Derivative Works thereof in any medium, with or without
+   modifications, and in Source or Object form, provided that You
+   meet the following conditions:
+
+   (a) You must give any other recipients of the Work or
+       Derivative Works a copy of this License; and
+
+   (b) You must cause any modified files to carry prominent notices
+       stating that You changed the files; and
+
+   (c) You must retain, in the Source form of any Derivative Works
+       that You distribute, all copyright, patent, trademark, and
+       attribution notices from the Source form of the Work,
+       excluding those notices that do not pertain to any part of
+       the Derivative Works; and
+
+   (d) If the Work includes a "NOTICE" text file as part of its
+       distribution, then any Derivative Works that You distribute must
+       include a readable copy of the attribution notices contained
+       within such NOTICE file, excluding those notices that do not
+       pertain to any part of the Derivative Works, in at least one
+       of the following places: within a NOTICE text file distributed
+       as part of the Derivative Works; within the Source form or
+       documentation, if provided along with the Derivative Works; or,
+       within a display generated by the Derivative Works, if and
+       wherever such third-party notices normally appear. The contents
+       of the NOTICE file are for informational purposes only and
+       do not modify the License. You may add Your own attribution
+       notices within Derivative Works that You distribute, alongside
+       or as an addendum to the NOTICE text from the Work, provided
+       that such additional attribution notices cannot be construed
+       as modifying the License.
+
+   You may add Your own copyright statement to Your modifications and
+   may provide additional or different license terms and conditions
+   for use, reproduction, or distribution of Your modifications, or
+   for any such Derivative Works as a whole, provided Your use,
+   reproduction, and distribution of the Work otherwise complies with
+   the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise,
+   any Contribution intentionally submitted for inclusion in the Work
+   by You to the Licensor shall be under the terms and conditions of
+   this License, without any additional terms or conditions.
+   Notwithstanding the above, nothing herein shall supersede or modify
+   the terms of any separate license agreement you may have executed
+   with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade
+   names, trademarks, service marks, or product names of the Licensor,
+   except as required for reasonable and customary use in describing the
+   origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or
+   agreed to in writing, Licensor provides the Work (and each
+   Contributor provides its Contributions) on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+   implied, including, without limitation, any warranties or conditions
+   of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+   PARTICULAR PURPOSE. You are solely responsible for determining the
+   appropriateness of using or redistributing the Work and assume any
+   risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory,
+   whether in tort (including negligence), contract, or otherwise,
+   unless required by applicable law (such as deliberate and grossly
+   negligent acts) or agreed to in writing, shall any Contributor be
+   liable to You for damages, including any direct, indirect, special,
+   incidental, or consequential damages of any character arising as a
+   result of this License or out of the use or inability to use the
+   Work (including but not limited to damages for loss of goodwill,
+   work stoppage, computer failure or malfunction, or any and all
+   other commercial damages or losses), even if such Contributor
+   has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing
+   the Work or Derivative Works thereof, You may choose to offer,
+   and charge a fee for, acceptance of support, warranty, indemnity,
+   or other liability obligations and/or rights consistent with this
+   License. However, in accepting such obligations, You may act only
+   on Your own behalf and on Your sole responsibility, not on behalf
+   of any other Contributor, and only if You agree to indemnify,
+   defend, and hold each Contributor harmless for any liability
+   incurred by, or claims asserted against, such Contributor by reason
+   of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+   To apply the Apache License to your work, attach the following
+   boilerplate notice, with the fields enclosed by brackets "[]"
+   replaced with your own identifying information. (Don't include
+   the brackets!)  The text should be enclosed in the appropriate
+   comment syntax for the file format. We also recommend that a
+   file or class name and description of purpose be included on the
+   same "printed page" as the copyright notice for easier
+   identification within third-party archives.
+
+Copyright [yyyy] [name of copyright owner]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,90 @@
+Knife Hadoop 
+===============
+
+This is a Chef Knife plugin for Hadoop. This plugin gives knife the ability to provision, list, and manage Hadoop for Operators. 
+
+Version 0.0.3
+
+Features:
+
+HDFS APIs (currently supported) using the ruby webhdfs gem: https://github.com/kzk/webhdfs. Extensions to webhdfs will be hosted at 
+https://github.com/murraju/webhdfs
+	
+	a. List Directories and Files
+	b. Snapshot metadata information to a database (PostgreSQL for now). Useful for reporting and audits
+	c. Create Directories and Files
+	d. Update Files
+	e. Read Files
+
+MapReduce APIs supported using the awesome work done by huahin: https://github.com/huahin
+
+	a. Start/List/Kill MapReduce Jobs by JobID and JobName
+
+
+
+Issues:
+
+1. The WebHDFS gem has bugs on net-http for create/delete
+2. Not all methods are exposed
+
+
+
+# Installation #
+
+Be sure you are running the latest version Chef. Versions earlier than 0.10.0 don't support plugins:
+
+    $ gem install chef
+
+This plugin is distributed as a Ruby Gem. To install it, run:
+
+    $ gem install knife-hadoop
+
+Depending on your system's configuration, you may need to run this command with root privileges.
+
+# Configuration #
+
+In order to communicate with Hadoop and other APIs, you will have to set parameters. The easiest way to accomplish this is to create some entries in your `knife.rb` file:
+
+	knife[:namenode_host]   	= "namenode"
+	knife[:namenode_port]   	= "port"
+	knife[:namenode_username] 	= "namenode_username"
+	knife[:mapred_mgmt_host]    = "mapred_mgmt_host"
+	knife[:mapred_mgmt_port]    = "mapred_mgmt_port"
+	knife[:db_username] 		= "dbusername"
+	knife[:db_password] 		= "dbpassword"
+	knife[:db_host] 			= "dbhost"
+	knife[:db] 					= "db"
+
+If your knife.rb file will be checked into a SCM system (ie readable by others) you may want to read the values from environment variables:
+
+	knife[:namenode_host]   	= "#{ENV['NAMENODE_HOST']}"
+	knife[:namenode_port]   	= "#{ENV['NAMENODE_PORT']}"
+	knife[:namenode_username] 	= "#{ENV['NAMENODE_USERNAME']}"
+	knife[:mapred_mgmt_host]    = "#{ENV['MAPRED_MGMT_HOST']}"
+	knife[:mapred_mgmt_port]    = "#{ENV['MAPRED_MGMT_PORT']}"
+	knife[:db_username] 		= "#{ENV['DB_USERNAME']}"
+	knife[:db_password] 		= "#{ENV['DB_PASSWORD']}"
+	knife[:db_host] 			= "#{ENV['DB_HOST']}"
+	knife[:db] 					= "#{ENV['DB']}"
+
+
+
+# License #
+
+Author:: Murali Raju <murali.raju@appliv.com>
+
+Copyright:: Copyright (c) 2012 Murali Raju <murali.raju@appliv.com>
+
+License:: Apache License, Version 2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/configs/hosfw.json

@@ -0,0 +1 @@
+

data/knife-hadoop.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "knife-hadoop/version"
+
+Gem::Specification.new do |s|
+  s.name        = "knife-hadoop"
+  s.version     = Knife::Hadoop::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.has_rdoc = true
+  s.extra_rdoc_files = ["README.md", "LICENSE" ]
+  s.authors     = ["Murali Raju"]
+  s.email       = ["murraju@appliv.com"]
+  s.homepage    = "https://github.com/murraju/knife-hadoop"
+  s.summary     = %q{Hadoop Chef Knife Plugin}
+  s.description = s.summary
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency "webhdfs"
+  s.add_dependency "pg"
+  s.add_dependency "sequel"
+  s.add_dependency "debugger"
+  s.add_dependency "rest-client"
+  s.add_dependency "chef"
+end

data/lib/chef/knife/hadoop_base.rb

@@ -0,0 +1,129 @@
+# Author:: Murali Raju (<murali.raju@appliv.com>)
+# Copyright:: Copyright (c) 2012 Murali Raju.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/knife'
+require 'webhdfs'
+require 'debugger'
+require 'sequel'
+require 'rest-client'
+require 'json'
+
+class Chef
+  class Knife
+    module HadoopBase
+
+      def self.included(includer)
+        includer.class_eval do
+
+          deps do
+            require 'readline'
+            require 'chef/json_compat'
+          end
+
+          option :namenode_username,
+            :short => "-U USERNAME",
+            :long => "--namenode-username USERNAME",
+            :description => "NameNode Username",
+            :proc => Proc.new { |key| Chef::Config[:knife][:namenode_username] = key }
+
+          option :namenode_password,
+            :short => "-P PASSWORD",
+            :long => "--namenode-password PASSWORD",
+            :description => "NameNode password",
+            :proc => Proc.new { |key| Chef::Config[:knife][:namenode_password] = key }
+
+          option :namenode_host,
+            :short => "-H HOST",
+            :long => "--namenode-host HOST",
+            :description => "NameNode FQDN or IP address",
+            :proc => Proc.new { |endpoint| Chef::Config[:knife][:namenode_host] = endpoint }
+
+          option :namenode_port,
+            :short => "-A PORT",
+            :long => "--namenode-port PORT",
+            :description => "NameNode port",
+            :proc => Proc.new { |key| Chef::Config[:knife][:namenode_port] = key }
+
+          option :db,
+            :short => "-D DATABASE",
+            :long => "--database DATABASE",
+            :description => "PostgreSQL Database to use for Hadoop Management Data",
+            :proc => Proc.new { |key| Chef::Config[:knife][:db] = key }
+
+          option :db_username,
+            :short => "-B DBUSERNAME",
+            :long => "--db-username DBUSERNAME",
+            :description => "PostgreSQL DB Username",
+            :proc => Proc.new { |key| Chef::Config[:knife][:db_username] = key }
+
+          option :db_password,
+            :short => "-C DBPASSWORD",
+            :long => "--db-password DBPASSWORD",
+            :description => "PostgreSQL DB Password",
+            :proc => Proc.new { |key| Chef::Config[:knife][:db_password] = key }
+
+          option :db_host,
+            :short => "-I DBHOST",
+            :long => "--db-host DBHOST",
+            :description => "PostgreSQL DB Host",
+            :proc => Proc.new { |key| Chef::Config[:knife][:db_host] = key }
+          
+        end
+      end
+
+      def hdfs_connection
+        Chef::Log.debug("username: #{Chef::Config[:knife][:namenode_username]}")
+        Chef::Log.debug("password: #{Chef::Config[:knife][:namenode_password]}")
+        Chef::Log.debug("host:     #{Chef::Config[:knife][:namenode_host]}")
+        Chef::Log.debug("port:     #{Chef::Config[:knife][:namenode_port]}")
+        @hdfs_connection ||= begin
+          hdfs_connection = WebHDFS::Client.new("#{Chef::Config[:knife][:namenode_host]}", 
+                                           "#{Chef::Config[:knife][:namenode_port]}",
+                                           "#{Chef::Config[:knife][:namenode_username]}")
+        end
+      end
+
+      def db_connection
+        Chef::Log.debug("db: #{Chef::Config[:knife][:db]}")
+        Chef::Log.debug("db_username: #{Chef::Config[:knife][:db_username]}")
+        Chef::Log.debug("db_password: #{Chef::Config[:knife][:db_password]}")
+        Chef::Log.debug("db_host: #{Chef::Config[:knife][:db_host]}")
+        @db_connection ||= begin
+          db_connection = Sequel.connect("postgres://#{Chef::Config[:knife][:db_username]}"+':'+
+                                         "#{Chef::Config[:knife][:db_password]}"+'@'+
+                                         "#{Chef::Config[:knife][:db_host]}"+'/'+ 
+                                         "#{Chef::Config[:knife][:db]}")
+        end
+      end
+
+      
+      def locate_config_value(key)
+        key = key.to_sym
+        Chef::Config[:knife][key] || config[key]
+      end
+
+      def msg_pair(label, value, color=:cyan)
+        if value && !value.to_s.empty?
+          puts "#{ui.color(label, color)}: #{value}"
+        end
+      end
+
+    end
+  end
+end
+
+

data/lib/chef/knife/hadoop_hdfs_create.rb

@@ -0,0 +1,97 @@
+# Author:: Murali Raju (<murali.raju@appliv.com>)
+# Copyright:: Copyright (c) 2012 Murali Raju.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'chef/knife/hadoop_base'
+
+class Chef
+  class Knife
+    class HadoopHdfsCreate < Knife
+
+      include Knife::HadoopBase
+
+      deps do
+        require 'readline'
+        require 'chef/json_compat'
+        require 'chef/knife/bootstrap'
+        Chef::Knife::Bootstrap.load_deps
+      end
+
+      banner "knife hadoop hdfs create (options)"
+
+      option :type,
+        :short => "-T TYPE",
+        :long => "--type TYPE",
+        :description => "The type <dir,file>",
+        :proc => Proc.new { |f| Chef::Config[:knife][:type] = f }      
+      
+      option :data,
+        :short => "-D DATA",
+        :long => "--data DATA",
+        :description => "The data to be populated into a file",
+        :proc => Proc.new { |f| Chef::Config[:knife][:data] = f }
+
+      option :path,
+        :short => "-P PATH",
+        :long => "--path PATH",
+        :description => "The HDFS path - Directory or File to create",
+        :proc => Proc.new { |f| Chef::Config[:knife][:path] = f }
+
+      option :overwrite,
+        :short => "-O OVERWRITE",
+        :long => "--overwrite OVERWRITE",
+        :description => "The overwrite bolean <true,fale>",
+        :proc => Proc.new { |f| Chef::Config[:knife][:overwrite] = f }
+
+      option :blocksize,
+        :short => "-B BLOCKSIZE",
+        :long => "--blocksize BLOCKSIZE",
+        :description => "The blocksize",
+        :proc => Proc.new { |f| Chef::Config[:knife][:blocksize] = f }
+
+      option :replication,
+        :short => "-R REPLICATION",
+        :long => "--replication REPLICATION",
+        :description => "The replication factor <n>",
+        :proc => Proc.new { |f| Chef::Config[:knife][:replication] = f }
+
+      option :permission,
+        :short => "-P PERM",
+        :long => "--permission PERMISSION",
+        :description => "The permissions of the directory",
+        :proc => Proc.new { |f| Chef::Config[:knife][:permission] = f }
+
+
+      def run
+        $stdout.sync = true
+      
+        type = "#{Chef::Config[:knife][:type]}".downcase
+        case type
+        when 'dir'
+          hdfs_connection.mkdir("#{Chef::Config[:knife][:path]}", :permission => "#{Chef::Config[:knife][:permission]}")
+        when 'file'
+          ; debugger
+          hdfs_connection.create("#{Chef::Config[:knife][:path]}", "#{Chef::Config[:knife][:data]}",
+                                 :overwrite => "#{Chef::Config[:knife][:overwrite]}", :blocksize => "#{Chef::Config[:knife][:blocksize]}",
+                                 :replication => "#{Chef::Config[:knife][:replication]}", :permission => "#{Chef::Config[:knife][:permission]}"
+                                 )
+        else
+          ui.error ("Incorrect options. Please use --help to list options.")
+        end
+      end
+    end
+  end
+end
+

data/lib/chef/knife/hadoop_hdfs_list.rb

@@ -0,0 +1,93 @@
+# Author:: Murali Raju (<murali.raju@appliv.com>)
+# Copyright:: Copyright (c) 2012 Murali Raju.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'chef/knife/hadoop_base'
+
+class Chef
+  class Knife
+    class HadoopHdfsList < Knife
+
+      include Knife::HadoopBase
+
+      deps do
+        require 'readline'
+        require 'chef/json_compat'
+        require 'chef/knife/bootstrap'
+        Chef::Knife::Bootstrap.load_deps
+      end
+
+      banner "knife hadoop hdfs list (options)"
+      
+      option :dir,
+        :short => "-D DIR",
+        :long => "--directory DIRECTORY",
+        :description => "The HDFS directory",
+        :proc => Proc.new { |f| Chef::Config[:knife][:dir] = f }
+
+      def run
+        $stdout.sync = true
+        
+        hdfs_list = [
+          ui.color('Directory',        :bold),
+          ui.color('accessTime',       :bold),
+          ui.color('blockSize',        :bold),
+          ui.color('group',            :bold),
+          ui.color('length',           :bold),
+          ui.color('modificationTime', :bold),
+          ui.color('owner',            :bold),
+          ui.color('pathSuffix',       :bold),
+          ui.color('permission',       :bold),
+          ui.color('replication',      :bold),
+          ui.color('type',             :bold)
+        ]
+        
+        #There has to be a more elegant way to do the below iteration :-)
+
+        hdfs_layout = hdfs_connection.list("#{Chef::Config[:knife][:dir]}") 
+        hdfs_layout.each do |item|
+          hdfs_list << "#{Chef::Config[:knife][:dir]}"
+          item.each do |k, v|
+            if "#{k}" == 'accessTime'
+               hdfs_list << item['accessTime'].to_s
+            elsif "#{k}" == 'blockSize'
+               hdfs_list << item['blockSize'].to_s
+            elsif "#{k}" == 'group'
+               hdfs_list << item['group'].to_s
+            elsif "#{k}" == 'length'
+               hdfs_list << item['length'].to_s
+            elsif "#{k}" == 'modificationTime'
+               hdfs_list << item['modificationTime'].to_s
+            elsif "#{k}" == 'owner'
+               hdfs_list << item['owner'].to_s
+            elsif "#{k}" == 'pathSuffix'
+               hdfs_list << item['pathSuffix'].to_s
+            elsif "#{k}" == 'permission'
+               hdfs_list << item['permission'].to_s
+            elsif "#{k}" == 'replication'
+               hdfs_list << item['replication'].to_s
+            elsif "#{k}" == 'type'
+               hdfs_list << item['type'].to_s
+            else
+              ui.error ("WebHDFS is not responding. Please debug")
+            end
+          end 
+        end
+        puts ui.list(hdfs_list, :uneven_columns_across, 11)
+      end
+    end
+  end
+end
+

data/lib/chef/knife/hadoop_hdfs_snapshot.rb

@@ -0,0 +1,99 @@
+# Author:: Murali Raju (<murali.raju@appliv.com>)
+# Copyright:: Copyright (c) 2012 Murali Raju.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'chef/knife/hadoop_base'
+
+class Chef
+  class Knife
+    class HadoopHdfsSnapshot < Knife
+
+      include Knife::HadoopBase
+
+      deps do
+        require 'readline'
+        require 'chef/json_compat'
+        require 'chef/knife/bootstrap'
+        Chef::Knife::Bootstrap.load_deps
+      end
+
+      banner "knife hadoop hdfs snapshot (options)"
+      
+      option :dir,
+        :short => "-D DIR",
+        :long => "--directory DIRECTORY",
+        :description => "The HDFS directory",
+        :proc => Proc.new { |f| Chef::Config[:knife][:dir] = f }
+
+      option :table,
+        :short => "-X TABLE",
+        :long => "--database-table TABLE",
+        :description => "The database table to be used",
+        :proc => Proc.new { |f| Chef::Config[:knife][:table] = f }
+
+      def run
+        $stdout.sync = true
+        dataset = db_connection.from("#{Chef::Config[:knife][:table]}")
+        list = hdfs_connection.list("#{Chef::Config[:knife][:dir]}")
+        list.each do |item|
+          item.each do |k, v|
+            if "#{k}" == 'accessTime'
+              @accessTime = item['accessTime']
+            elsif "#{k}" == 'blockSize'
+              @blockSize = item['blockSize']
+            elsif "#{k}" == 'group'
+              @group = item['group']
+            elsif "#{k}" == 'length'
+              @length = item['length']
+            elsif "#{k}" == 'modificationTime'
+              @modificationTime = item['modificationTime']
+            elsif "#{k}" == 'owner'
+              @owner = item['owner']
+            elsif "#{k}" == 'pathSuffix'
+              @pathSuffix = item['pathSuffix']
+            elsif "#{k}" == 'permission'
+              @permission = item['permission']
+            elsif "#{k}" == 'replication'
+              @replication = item['replication']
+            elsif "#{k}" == 'type'
+              @type = item['type']
+            else
+              puts "Cannot read key value pairs. Please debug"
+            end
+          end
+
+          db_connection.transaction do
+                dataset.insert(
+                  :directory => "#{Chef::Config[:knife][:dir]}",
+                  :accessTime => "#{@accessTime}",
+                  :blockSize =>  "#{@blockSize}",
+                  :group =>  "#{@group}",
+                  :length =>  "#{@length}",
+                  :modificationTime =>  "#{@modificationTime}",
+                  :owner =>  "#{@owner}",
+                  :pathSuffix =>  "#{@pathSuffix}",
+                  :permission =>  "#{@permission}",
+                  :replication => "#{@replication}",
+                  :type =>  "#{@type}",
+                  :created_at => @created_at = Time.now
+                  ) 
+          end      
+        end
+        ui.msg ("Created snapshot in #{Chef::Config[:knife][:table]} at #{@created_at}")         
+      end
+    end
+  end
+end
+

data/lib/chef/knife/hadoop_hdfs_update.rb

@@ -0,0 +1,57 @@
+# Author:: Murali Raju (<murali.raju@appliv.com>)
+# Copyright:: Copyright (c) 2012 Murali Raju.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'chef/knife/hadoop_base'
+
+class Chef
+  class Knife
+    class HadoopHdfsUpdate < Knife
+
+      include Knife::HadoopBase
+
+      deps do
+        require 'readline'
+        require 'chef/json_compat'
+        require 'chef/knife/bootstrap'
+        Chef::Knife::Bootstrap.load_deps
+      end
+
+      banner "knife hadoop hdfs update (options)"
+   
+      
+      option :path,
+        :short => "-O ORIGINALPATH",
+        :long => "--original-path ORIGINALPATH",
+        :description => "The original HDFS path",
+        :proc => Proc.new { |f| Chef::Config[:knife][:path] = f }
+
+      option :newpath,
+        :short => "-N NEWPATH",
+        :long => "--new-path NEWPATH",
+        :description => "The new HDFS path",
+        :proc => Proc.new { |f| Chef::Config[:knife][:newpath] = f }
+
+
+      def run
+        $stdout.sync = true
+      
+        hdfs_connection.rename("#{Chef::Config[:knife][:path]}", 
+                               "#{Chef::Config[:knife][:newpath]}")
+      end
+    end
+  end
+end
+

data/lib/chef/knife/hadoop_mapred_job_kill.rb

@@ -0,0 +1,69 @@
+# Author:: Murali Raju (<murali.raju@appliv.com>)
+# Copyright:: Copyright (c) 2012 Murali Raju.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'chef/knife/hadoop_base'
+
+class Chef
+  class Knife
+    class HadoopMapredJobKill < Knife
+
+      include Knife::HadoopBase
+
+      deps do
+        require 'readline'
+        require 'chef/json_compat'
+        require 'chef/knife/bootstrap'
+        Chef::Knife::Bootstrap.load_deps
+      end
+
+      banner "knife hadoop mapred job kill (options)"
+
+      option :filter,
+        :short => "-F FILTER",
+        :long => "--filter FILTER",
+        :description => "Kill Job by either <id,name>",
+        :proc => Proc.new { |f| Chef::Config[:knife][:filter] = f }
+      
+      option :jobid,
+        :short => "-J JOBID",
+        :long => "--job-id JOBID",
+        :description => "The MapReduce JobID",
+        :proc => Proc.new { |f| Chef::Config[:knife][:jobid] = f }
+
+      option :jobname,
+        :short => "-N JOBNAME",
+        :long => "--job-name JOBNAME",
+        :description => "The MapReduce JobNAME",
+        :proc => Proc.new { |f| Chef::Config[:knife][:jobname] = f }
+
+      def run
+        $stdout.sync = true
+
+        Chef::Log.debug("username: #{Chef::Config[:knife][:mapred_mgmt_host]}")
+        Chef::Log.debug("password: #{Chef::Config[:knife][:mapred_mgmt_port]}")
+
+        filter = "#{Chef::Config[:knife][:filter]}".downcase
+        case filter
+        when 'id'
+          RestClient.delete "http://#{Chef::Config[:knife][:mapred_mgmt_host]}:#{Chef::Config[:knife][:mapred_mgmt_port]}/job/kill/id/#{Chef::Config[:knife][:jobid]}"
+        when 'name'
+          RestClient.delete "http://#{Chef::Config[:knife][:mapred_mgmt_host]}:#{Chef::Config[:knife][:mapred_mgmt_port]}/job/kill/name/#{Chef::Config[:knife][:jobname]}"
+        end
+      end
+    end
+  end
+end
+

data/lib/chef/knife/hadoop_mapred_job_list.rb

@@ -0,0 +1,102 @@
+# Author:: Murali Raju (<murali.raju@appliv.com>)
+# Copyright:: Copyright (c) 2012 Murali Raju.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'chef/knife/hadoop_base'
+
+class Chef
+  class Knife
+    class HadoopMapredJobList < Knife
+
+      include Knife::HadoopBase
+
+      deps do
+        require 'readline'
+        require 'chef/json_compat'
+        require 'chef/knife/bootstrap'
+        Chef::Knife::Bootstrap.load_deps
+      end
+
+      banner "knife hadoop mapred job list (options)"
+      
+      option :filter,
+        :short => "-F FILTER",
+        :long => "--filter FILTER",
+        :description => "List MapReduce jobs <all,user>",
+        :proc => Proc.new { |f| Chef::Config[:knife][:filter] = f }
+
+      option :name,
+        :short => "-N NAME",
+        :long => "--name NAME",
+        :description => "List by User Name",
+        :proc => Proc.new { |f| Chef::Config[:knife][:name] = f }
+
+      def run
+        $stdout.sync = true
+
+        Chef::Log.debug("username: #{Chef::Config[:knife][:mapred_mgmt_host]}")
+        Chef::Log.debug("password: #{Chef::Config[:knife][:mapred_mgmt_port]}")
+
+        job_list = [
+          ui.color('jobid',           :bold),
+          ui.color('mapComplete',     :bold),
+          ui.color('name',            :bold),
+          ui.color('priority',        :bold),
+          ui.color('reduceComplete',  :bold),
+          ui.color('schedulingInfo',  :bold),
+          ui.color('startTime',       :bold),
+          ui.color('state',           :bold),
+          ui.color('user',            :bold)
+        ]
+        
+        filter = "#{Chef::Config[:knife][:filter]}".downcase
+        case filter
+        when 'all'
+          response = RestClient.get "http://#{Chef::Config[:knife][:mapred_mgmt_host]}:#{Chef::Config[:knife][:mapred_mgmt_port]}/job/list"
+          collection = JSON.parse(response)
+          collection.each do |item|
+            job_list << item['jobid']
+            job_list << item['mapComplete']
+            job_list << item['name']
+            job_list << item['priority']
+            job_list << item['reduceComplete']
+            job_list << item['schedulingInfo']
+            job_list << item['startTime']
+            job_list << item['state']
+            job_list << item['user']
+          end
+        when 'user'
+          response = RestClient.get "http://#{Chef::Config[:knife][:mapred_mgmt_host]}:#{Chef::Config[:knife][:mapred_mgmt_port]}/job/list"
+          collection = JSON.parse(response)
+          collection.each do |item|
+           if "#{Chef::Config[:knife][:name]}" == item['user']
+              job_list << item['jobid']
+              job_list << item['mapComplete']
+              job_list << item['name']
+              job_list << item['priority']
+              job_list << item['reduceComplete']
+              job_list << item['schedulingInfo']
+              job_list << item['startTime']
+              job_list << item['state']
+              job_list << item['user']
+            end
+          end
+        end
+        puts ui.list(job_list, :uneven_columns_across, 9)
+      end
+    end
+  end
+end
+

data/lib/chef/knife/hadoop_setup.rb

@@ -0,0 +1,73 @@
+# Author:: Murali Raju (<murali.raju@appliv.com>)
+# Copyright:: Copyright (c) 2012 Murali Raju.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'chef/knife/hadoop_base'
+
+class Chef
+  class Knife
+    class HadoopSetup < Knife
+
+      include Knife::HadoopBase
+
+      deps do
+        require 'readline'
+        require 'chef/json_compat'
+        require 'chef/knife/bootstrap'
+        Chef::Knife::Bootstrap.load_deps
+      end
+
+      banner "knife hadoop setup (options)"
+
+      option :type,
+        :short => "-T TYPE",
+        :long => "--setup-type SETUP TYPE",
+        :description => "The setup type <db,cluster>",
+        :proc => Proc.new { |f| Chef::Config[:knife][:type] = f }
+      
+
+      option :table,
+        :short => "-X TABLE",
+        :long => "--database-table TABLE",
+        :description => "The database table to be created",
+        :proc => Proc.new { |f| Chef::Config[:knife][:table] = f }
+
+      def run
+        $stdout.sync = true
+    
+        type = "#{Chef::Config[:knife][:type]}".downcase
+        case type
+        when 'db'
+          ui.msg("Creating table #{Chef::Config[:knife][:table]}")
+          db_connection.create_table "#{Chef::Config[:knife][:table]}" do
+             String :directory
+             String :accessTime
+             String :blockSize
+             String :group
+             String :length
+             String :modificationTime
+             String :owner
+             String :pathSuffix
+             String :permission
+             String :replication
+             String :type
+             DateTime :created_at      
+          end          
+        end
+      end
+    end
+  end
+end
+

data/lib/knife-hadoop/version.rb

@@ -0,0 +1,6 @@
+module Knife
+  module Hadoop
+    VERSION = "0.0.4"
+    MAJOR, MINOR, TINY = VERSION.split('.')
+  end
+end

metadata

@@ -0,0 +1,159 @@
+--- !ruby/object:Gem::Specification
+name: knife-hadoop
+version: !ruby/object:Gem::Version
+  version: 0.0.4
+  prerelease: 
+platform: ruby
+authors:
+- Murali Raju
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-02-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: webhdfs
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: debugger
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: chef
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Hadoop Chef Knife Plugin
+email:
+- murraju@appliv.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.md
+- LICENSE
+files:
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- configs/hosfw.json
+- knife-hadoop.gemspec
+- lib/chef/knife/hadoop_base.rb
+- lib/chef/knife/hadoop_hdfs_create.rb
+- lib/chef/knife/hadoop_hdfs_list.rb
+- lib/chef/knife/hadoop_hdfs_snapshot.rb
+- lib/chef/knife/hadoop_hdfs_update.rb
+- lib/chef/knife/hadoop_mapred_job_kill.rb
+- lib/chef/knife/hadoop_mapred_job_list.rb
+- lib/chef/knife/hadoop_setup.rb
+- lib/knife-hadoop/version.rb
+homepage: https://github.com/murraju/knife-hadoop
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Hadoop Chef Knife Plugin
+test_files: []