qubole 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 67c203cfa081fa7f902b5f55b9b9525e0ea990a7
-  data.tar.gz: 311e01beb229fc56270f5a8272c0c57bdeb25f4e
+  metadata.gz: 5dfda23489fb76ffd03f8c6ee148671132eb2453
+  data.tar.gz: 3cb9c779a321db0873d610732d02a9e00f7d0738
 SHA512:
-  metadata.gz: fb368b40b47aa8d37b6bbb0bc9128228fc93e3a1076da4ff1e7378cd3656992e75fb7036f247411c56faf38197b63b847d8978b0e828a54916a255577e28c24b
-  data.tar.gz: 26dedf8fd4b6dde70f6c38ebda6149b5e53c1bf7c314ff68b0d75b35f24e2a2ae24d375f39ed90588f97af61f6a89db0974d137e0cfa06bec1d731095ddacc52
+  metadata.gz: 845c89cff18eef9c338d7088c418f9ed590eaa5d550593fbcefaae6cc8bbe351d05622766c417d97cab658431adb28926ca0123853a2998945e090f9906f579e
+  data.tar.gz: 73ed357a515370cdf2011938b05316945c1e8dc10a5b2ce06b6b1ddc3e17c32b0ea4f6f83af32c3a0245e41f7e9e3184bb30eb335e7edbe9d1c99bd12a39db7c

data/README.md

@@ -5,6 +5,7 @@ Qubole REST API client
 [![Build Status](https://travis-ci.org/Demandbase/qubole-ruby.svg?branch=master)](https://travis-ci.org/Demandbase/qubole-ruby)
 [![Code Climate](https://codeclimate.com/github/Demandbase/qubole/badges/gpa.svg)](https://codeclimate.com/github/Demandbase/qubole)
 [![Test Coverage](https://codeclimate.com/github/Demandbase/qubole/badges/coverage.svg)](https://codeclimate.com/github/Demandbase/qubole/coverage)
+[![Gem Version](https://badge.fury.io/rb/qubole.svg)](http://badge.fury.io/rb/qubole)
 
 ## Documentation
 
@@ -95,25 +96,25 @@ Or install it yourself as:
 	- [ ] All Commands Report
 	- [ ] Canonical Hive Commands Report
 	- [ ] Cluster nodes Report
-- [ ] Cluster API
-	- [ ] List All Clusters
-	- [ ] Create a New Cluster
-	- [ ] View Cluster Configuration
-	- [ ] Edit Cluster Configuration
-	- [ ] Clone a Cluster
-	- [ ] Start or Terminate a Cluster
-	- [ ] Check Cluster Status
-	- [ ] Delete a Cluster
-	- [ ] Reassign Cluster Label
-	- [ ] Run Adhoc Scripts on a Cluster
-	- [ ] Cluster Metrics
-	- [ ] Add a Node to a Cluster
-	- [ ] Replace a Node in a Cluster
-	- [ ] Remove a Node from a Cluster
-	- [ ] Take an HBase Snapshot
-	- [ ] View an HBase Snapshot Schedule
-	- [ ] Restore HBase Tables
-	- [ ] Update an HBase Snapshot Schedule
+- [x] Cluster API
+	- [x] List All Clusters
+	- [x] Create a New Cluster
+	- [x] View Cluster Configuration
+	- [x] Edit Cluster Configuration
+	- [x] Clone a Cluster
+	- [x] Start or Terminate a Cluster
+	- [x] Check Cluster Status
+	- [x] Delete a Cluster
+	- [x] Reassign Cluster Label
+	- [x] Run Adhoc Scripts on a Cluster
+	- [x] Cluster Metrics
+	- [x] Add a Node to a Cluster
+	- [x] Replace a Node in a Cluster
+	- [x] Remove a Node from a Cluster
+	- [x] Take an HBase Snapshot
+	- [x] View an HBase Snapshot Schedule
+	- [x] Restore HBase Tables
+	- [x] Update an HBase Snapshot Schedule
 
 
 ## Contributing

data/lib/qubole.rb

@@ -1,6 +1,7 @@
 require "qubole/version"
 require "qubole/exceptions"
 require "qubole/commands"
+require "qubole/cluster"
 require "net/http"
 require "json"
 

data/lib/qubole/cluster.rb

@@ -0,0 +1,134 @@
+module Qubole
+
+  # To look for what all parameters need to be passed go to http://docs.qubole.com/en/latest/rest-api/cluster_api/
+  # Pass the params as hash from above doc link
+  class Cluster
+
+    # List existing clusters present in your account.
+    def self.list
+      Qubole.get("/clusters")
+    end
+
+    # Show information about the cluster with id/label `cluster_id_label`.
+    def self.show(cluster_id_label)
+      Qubole.get("/clusters/#{cluster_id_label}")
+    end
+
+    # Create a new cluster
+    # http://docs.qubole.com/en/latest/rest-api/cluster_api/create-new-cluster.html
+    def self.create(params={})
+     Qubole.post("/clusters", params)
+    end
+
+    # Update the cluster with id/label `cluster_id_label` using information provided in `cluster_info`.
+    # Assign id/label of cluster to id
+    def self.update(id, params={})
+      Qubole.put("/clusters/#{id}", params)
+    end
+
+    # Clone the cluster with id/label `cluster_id_label` using information provided in `cluster_info`.
+    # Assign id/label of cluster to id
+    def self.clone(id, params={})
+      Qubole.post("/clusters/#{id}/clone", params)
+    end
+
+    # Start the cluster with id/label.
+    def self.start(id)
+      params = {}
+      params[:state] = "start"
+      Qubole.put("/clusters/#{id}/state", {:state => "start"})
+    end
+
+    # Show the status of the cluster with id/label.
+    def self.status(id)
+      Qubole.get("/clusters/#{id}/state")
+    end
+
+    # Terminate the cluster with id/label `cluster_id_label`.
+    def self.terminate(id)
+      Qubole.put("/clusters/#{id}/state", {:state => "terminate"})
+    end
+
+    # Delete the cluster with id/label.
+    def self.delete(id)
+      Qubole.delete("/clusters/#{id}")
+    end
+
+    # Reassign a label from one cluster to another.
+    # http://docs.qubole.com/en/latest/rest-api/cluster_api/reassign-cluster-label.html#parameters
+    def self.reassign_label(params={})
+      Qubole.put("/clusters/reassign-label", params)
+    end
+
+    # Run an adhoc script
+    # http://docs.qubole.com/en/latest/rest-api/cluster_api/run-adhoc-scripts-cluster.html
+    # Assign id/label of cluster to id.
+    def self.run_script(id, params={})
+      Qubole.put("/clusters/#{id}/runscript.json", params)
+    end
+
+    # To look for cluster metrics
+    # http://docs.qubole.com/en/latest/rest-api/cluster_api/metrics-cluster.html
+    # Assign id/label of cluster to id.
+    def self.metrics(id, params={})
+      Qubole.get("/clusters/#{id}/metrics", params)
+    end
+
+    # To change the minimum and maximum size of a running cluster
+    # http://docs.qubole.com/en/latest/rest-api/cluster_api/resize-cluster.html
+    # Assign id/label of cluster to id.
+    def self.resize(id, params={})
+      Qubole.put("/clusters/#{id}",params)
+    end
+
+    # Add a node to an existing cluster
+    # http://docs.qubole.com/en/latest/rest-api/cluster_api/add-node.html
+    # Assign id/label of cluster to id.
+    def self.add_node(id, params={})
+      Qubole.post("/clusters/#{id}/nodes", params)
+    end
+
+    # Replaces a slave node in a cluster
+    # http://docs.qubole.com/en/latest/rest-api/cluster_api/replace-node.html
+    # Assign id/label of cluster to id.
+    def self.replace_node(id, params={})
+      params["command"] = "replace"
+      Qubole.put("/clusters/#{id}/nodes", params)
+    end
+
+    # Removes a slave node from a cluster
+    # http://docs.qubole.com/en/latest/rest-api/cluster_api/remove-node.html#parameters
+    # Assign id/label of cluster to id
+    def self.remove_node(id, params={})
+      Qubole.delete("/clusters/#{id}/nodes", params)
+    end
+
+    # Create hbase snapshot full/incremental
+    # http://docs.qubole.com/en/latest/rest-api/cluster_api/hbase-snapshot.html#parameters
+    # Assign id/label of cluster to id
+    def self.snapshot(id, params={})
+      Qubole.post("/clusters/#{id}/snapshots", params)
+    end
+
+    # Get details for snapshot schedule
+    def self.get_snapshot_schedule(id)
+      Qubole.get("/clusters/#{id}/snapshot_schedule")
+    end
+
+    # Restoring cluster from a given hbase snapshot id
+    # http://docs.qubole.com/en/latest/rest-api/cluster_api/restore-point.html#parameters
+    # Assign id/label of cluster to id
+    def self.restore_point(id, params={})
+      Qubole.post("/clusters/#{id}/restore_point", params)
+    end
+
+    # Update for snapshot schedule
+    # http://docs.qubole.com/en/latest/rest-api/cluster_api/update-snapshot-schedule.html#parameters
+    # Assign id/label of cluster to id
+    def self.update_snapshot_schedule(id, params={})
+      Qubole.put("/clusters/#{id}/snapshot_schedule", params)
+    end
+
+  end
+
+end

data/lib/qubole/version.rb

@@ -1,3 +1,3 @@
 module Qubole
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/qubole.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.email         = ["vladnik@demandbase.com"]
   spec.summary       = %q{Qubole REST API client}
   spec.description   = %q{Communication with Qubole REST API in a Ruby way}
-  spec.homepage      = "https://github.com/Demandbase/qubole"
+  spec.homepage      = "https://github.com/Demandbase/qubole-ruby"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files -z`.split("\x0")

data/spec/qubole/cluster_spec.rb

@@ -0,0 +1,218 @@
+module Qubole
+
+  describe Cluster do
+
+    before(:each) do
+      Qubole.stub(:http) do |name, path, param|
+        @request_type = name
+        @url_path = path
+        @p = param
+      end
+    end
+
+    describe "Cluster list" do
+      it "should return correct http arguments" do
+        Cluster.list
+        expect(@request_type).to eq("GET")
+        expect(@url_path).to eq("/clusters")
+        expect(@p).to eq({})
+      end
+    end
+
+    describe "Show cluster details" do
+      it "should return correct http arguments" do
+        Cluster.show(1)
+        expect(@request_type).to eq("GET")
+        expect(@url_path).to eq("/clusters/1")
+        expect(@p).to eq({})
+      end
+    end
+
+    describe "Create cluster" do
+      it "should return correct http arguments" do
+        Cluster.create({"cluster"=>{:label => ["my_cluster"], "node_configuration" => {"slave_request_type"=>"spot", "initial_nodes"=>1,
+                       "spot_instance_settings"=>{"timeout_for_request"=>10, "maximum_spot_instance_percentage"=>60,
+                       "maximum_bid_price_percentage"=>"100.0"}, "max_nodes"=>10, "master_instance_type"=>"m1.large", "slave_instance_type"=>
+                       "m1.xlarge"}, "hadoop_settings"=>{"use_hadoop2"=>false, "max_nodes"=>10, "fairscheduler_settings"=>{"default_pool"=>nil},
+                       "custom_config"=>nil}, "enable_ganglia_monitoring"=>false, "state"=>"DOWN", "node_bootstrap_file"=>"node_bootstrap.sh",
+                       "use_hadoop2"=>false, "security_settings"=>{"encrypted_ephemerals"=>false}, "ec2_settings"=>{"compute_validated"=>true,
+                       "aws_region"=>"us-west-2", "aws_preferred_availability_zone"=>"Any", "compute_secret_key"=>"xxxx",
+                       "vpc_id"=>nil, "compute_access_key"=>"xxxx", "subnet_id"=>nil}, "presto_settings"=>
+                       {"enable_presto"=>false, "custom_config"=>nil}, "disallow_cluster_termination"=>false}})
+
+        expect(@request_type).to eq("POST")
+        expect(@url_path).to eq("/clusters")
+        expect(@p).to eq({"cluster"=>{:label => ["my_cluster"], "node_configuration" => {"slave_request_type"=>"spot", "initial_nodes"=>1,
+                         "spot_instance_settings"=>{"timeout_for_request"=>10, "maximum_spot_instance_percentage"=>60,
+                         "maximum_bid_price_percentage"=>"100.0"}, "max_nodes"=>10, "master_instance_type"=>"m1.large", "slave_instance_type"=>
+                         "m1.xlarge"}, "hadoop_settings"=>{"use_hadoop2"=>false, "max_nodes"=>10, "fairscheduler_settings"=>{"default_pool"=>nil},
+                         "custom_config"=>nil}, "enable_ganglia_monitoring"=>false, "state"=>"DOWN", "node_bootstrap_file"=>"node_bootstrap.sh",
+                         "use_hadoop2"=>false, "security_settings"=>{"encrypted_ephemerals"=>false}, "ec2_settings"=>{"compute_validated"=>true, "aws_region"=>"us-west-2", "aws_preferred_availability_zone"=>"Any", "compute_secret_key"=>"xxxx",
+                         "vpc_id"=>nil, "compute_access_key"=>"xxxx", "subnet_id"=>nil}, "presto_settings"=>
+                         {"enable_presto"=>false, "custom_config"=>nil}, "disallow_cluster_termination"=>false}})
+
+      end
+    end
+
+
+    describe "Update cluster" do
+      it "label should return correct http arguments" do
+        Cluster.update(1, {'cluster' => { 'label' => ['test_label']}})
+        expect(@request_type).to eq("PUT")
+        expect(@url_path).to eq("/clusters/1")
+        expect(@p).to eq({'cluster' => { 'label' => ['test_label']}})
+      end
+    end
+
+    describe "Clone cluster" do
+      it "with label should return correct http arguments" do
+        Cluster.clone(1, {"cluster" => {"label" => ["test_label1", "test_label2"]}})
+        expect(@request_type).to eq("POST")
+        expect(@url_path).to eq("/clusters/1/clone")
+        expect(@p).to eq({"cluster" => {"label" => ["test_label1", "test_label2"]}})
+      end
+    end
+
+    describe "Start Cluster" do
+      it "should return correct http arguments" do
+        Cluster.start(1)
+        expect(@request_type).to eq("PUT")
+        expect(@url_path).to eq("/clusters/1/state")
+        expect(@p).to eq({:state => "start"})
+      end
+    end
+
+    describe "Status of cluster" do
+      it "should return correct http arguments" do
+        Cluster.status(1)
+        expect(@request_type).to eq("GET")
+        expect(@url_path).to eq("/clusters/1/state")
+        expect(@p).to eq({})
+      end
+    end
+
+    describe "Terminate Cluster" do
+      it "should return correct http arguments" do
+        Cluster.terminate(1)
+        expect(@request_type).to eq("PUT")
+        expect(@url_path).to eq("/clusters/1/state")
+        expect(@p).to eq({:state => "terminate"})
+      end
+    end
+
+    describe "Delete cluster" do
+      it "should return correct http arguments" do
+        Cluster.delete(1)
+        expect(@request_type).to eq("DELETE")
+        expect(@url_path).to eq("/clusters/1")
+        expect(@p).to eq({})
+      end
+    end
+
+    describe "Reassign label" do
+      it "should return correct http arguments" do
+        Cluster.reassign_label({'destination_cluster' => '123', 'label' => 'test_label'})
+        expect(@request_type).to eq("PUT")
+        expect(@url_path).to eq("/clusters/reassign-label")
+        expect(@p).to eq({'destination_cluster' => '123', 'label' => 'test_label'})
+      end
+    end
+
+    describe "Run script" do
+      it "using s3 path should return correct http arguments" do
+        Cluster.run_script(1, {"script" => "s3_path"})
+        expect(@request_type).to eq("PUT")
+        expect(@url_path).to eq("/clusters/1/runscript.json")
+        expect(@p).to eq({"script" => "s3_path"})
+      end
+    end
+
+    describe "Cluster metrics" do
+      it "should return correct http arguments" do
+        Cluster.metrics(1, {:metric => "mapred.jobtracker.maps_launched", :hostname => "master", :interval => "hour"})
+        expect(@request_type).to eq("GET")
+        expect(@url_path).to eq("/clusters/1/metrics")
+        expect(@p).to eq({:metric => "mapred.jobtracker.maps_launched", :hostname => "master", :interval => "hour"})
+      end
+    end
+
+    describe "Resize cluster" do
+      it "should return correct http arguments" do
+        Cluster.resize(1, {"node_configuration" => {"initial_nodes" => "3"}, "push" => "true"})
+        expect(@request_type).to eq("PUT")
+        expect(@url_path).to eq("/clusters/1")
+        expect(@p).to eq({"node_configuration" => {"initial_nodes" => "3"}, "push" => "true"})
+      end
+    end
+
+    describe "Add node to cluster" do
+      it "should return correct http arguments" do
+        Cluster.add_node(1, {"node_count" => "2"})
+        expect(@request_type).to eq("POST")
+        expect(@url_path).to eq("/clusters/1/nodes")
+        expect(@p).to eq({"node_count" => "2"})
+      end
+    end
+
+    describe "Replace node" do
+      it "should return correct http arguments" do
+        Cluster.replace_node(1, { "private_dns" => "ip-address"})
+        expect(@request_type).to eq("PUT")
+        expect(@url_path).to eq("/clusters/1/nodes")
+        expect(@p).to eq({ "private_dns" => "ip-address", "command" => "replace"})
+      end
+    end
+
+    describe "Remove node" do
+      it "should return correct http arguments" do
+        Cluster.remove_node(1, {"private_dns" => "ip-address"})
+        expect(@request_type).to eq("DELETE")
+        expect(@url_path).to eq("/clusters/1/nodes")
+        expect(@p).to eq({"private_dns" => "ip-address"})
+      end
+    end
+
+    describe "Create Hbase Snapshot" do
+      it "with no back up type should return correct http arguments" do
+        Cluster.snapshot(1, {"s3_location" => "s3-path"})
+        expect(@request_type).to eq("POST")
+        expect(@url_path).to eq("/clusters/1/snapshots")
+        expect(@p).to eq({"s3_location" => "s3-path"})
+      end
+    end
+
+    describe "Get snapshot schedule" do
+      it "should return correct http arguments" do
+        Cluster.get_snapshot_schedule(1)
+        expect(@request_type).to eq("GET")
+        expect(@url_path).to eq("/clusters/1/snapshot_schedule")
+        expect(@p).to eq({})
+      end
+    end
+
+    describe "Restore point" do
+      it "should return correct http arguments" do
+        Cluster.restore_point(1, { "table_names" => "test", "backup_id" => "abcd", "automatic" => true,
+          "overwrite" => true, "s3_location" => "s3-path" })
+        expect(@request_type).to eq("POST")
+        expect(@url_path).to eq("/clusters/1/restore_point")
+        expect(@p).to eq({ "table_names" => "test", "backup_id" => "abcd", "automatic" => true,
+                           "overwrite" => true, "s3_location" => "s3-path" })
+
+      end
+    end
+
+    describe "Update snapshot schedule" do
+      it "should return correct http arguments" do
+        Cluster.update_snapshot_schedule(1, {"snapshot-schedule" => { "frequency_unit" => "hours",
+                                        "frequency_num" => 20, "s3_location" => "s3-path"}})
+        expect(@request_type).to eq("PUT")
+        expect(@url_path).to eq("/clusters/1/snapshot_schedule")
+        expect(@p).to eq({"snapshot-schedule" => { "frequency_unit" => "hours",
+                         "frequency_num" => 20, "s3_location" => "s3-path"}})
+
+      end
+    end
+
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: qubole
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Volodymyr Ladnik
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-07 00:00:00.000000000 Z
+date: 2016-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -39,6 +39,7 @@ files:
 - README.md
 - Rakefile
 - lib/qubole.rb
+- lib/qubole/cluster.rb
 - lib/qubole/command.rb
 - lib/qubole/commands.rb
 - lib/qubole/commands/composite.rb
@@ -54,6 +55,7 @@ files:
 - lib/qubole/exceptions.rb
 - lib/qubole/version.rb
 - qubole.gemspec
+- spec/qubole/cluster_spec.rb
 - spec/qubole/command_spec.rb
 - spec/qubole/commands/composite_spec.rb
 - spec/qubole/commands/db_export_spec.rb
@@ -68,7 +70,7 @@ files:
 - spec/qubole/exceptions_spec.rb
 - spec/qubole_spec.rb
 - spec/spec_helper.rb
-homepage: https://github.com/Demandbase/qubole
+homepage: https://github.com/Demandbase/qubole-ruby
 licenses:
 - MIT
 metadata: {}
@@ -93,6 +95,7 @@ signing_key:
 specification_version: 4
 summary: Qubole REST API client
 test_files:
+- spec/qubole/cluster_spec.rb
 - spec/qubole/command_spec.rb
 - spec/qubole/commands/composite_spec.rb
 - spec/qubole/commands/db_export_spec.rb