qubole 0.0.1 → 0.0.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 67c203cfa081fa7f902b5f55b9b9525e0ea990a7
4
- data.tar.gz: 311e01beb229fc56270f5a8272c0c57bdeb25f4e
3
+ metadata.gz: 5dfda23489fb76ffd03f8c6ee148671132eb2453
4
+ data.tar.gz: 3cb9c779a321db0873d610732d02a9e00f7d0738
5
5
  SHA512:
6
- metadata.gz: fb368b40b47aa8d37b6bbb0bc9128228fc93e3a1076da4ff1e7378cd3656992e75fb7036f247411c56faf38197b63b847d8978b0e828a54916a255577e28c24b
7
- data.tar.gz: 26dedf8fd4b6dde70f6c38ebda6149b5e53c1bf7c314ff68b0d75b35f24e2a2ae24d375f39ed90588f97af61f6a89db0974d137e0cfa06bec1d731095ddacc52
6
+ metadata.gz: 845c89cff18eef9c338d7088c418f9ed590eaa5d550593fbcefaae6cc8bbe351d05622766c417d97cab658431adb28926ca0123853a2998945e090f9906f579e
7
+ data.tar.gz: 73ed357a515370cdf2011938b05316945c1e8dc10a5b2ce06b6b1ddc3e17c32b0ea4f6f83af32c3a0245e41f7e9e3184bb30eb335e7edbe9d1c99bd12a39db7c
data/README.md CHANGED
@@ -5,6 +5,7 @@ Qubole REST API client
5
5
  [![Build Status](https://travis-ci.org/Demandbase/qubole-ruby.svg?branch=master)](https://travis-ci.org/Demandbase/qubole-ruby)
6
6
  [![Code Climate](https://codeclimate.com/github/Demandbase/qubole/badges/gpa.svg)](https://codeclimate.com/github/Demandbase/qubole)
7
7
  [![Test Coverage](https://codeclimate.com/github/Demandbase/qubole/badges/coverage.svg)](https://codeclimate.com/github/Demandbase/qubole/coverage)
8
+ [![Gem Version](https://badge.fury.io/rb/qubole.svg)](http://badge.fury.io/rb/qubole)
8
9
 
9
10
  ## Documentation
10
11
 
@@ -95,25 +96,25 @@ Or install it yourself as:
95
96
  - [ ] All Commands Report
96
97
  - [ ] Canonical Hive Commands Report
97
98
  - [ ] Cluster nodes Report
98
- - [ ] Cluster API
99
- - [ ] List All Clusters
100
- - [ ] Create a New Cluster
101
- - [ ] View Cluster Configuration
102
- - [ ] Edit Cluster Configuration
103
- - [ ] Clone a Cluster
104
- - [ ] Start or Terminate a Cluster
105
- - [ ] Check Cluster Status
106
- - [ ] Delete a Cluster
107
- - [ ] Reassign Cluster Label
108
- - [ ] Run Adhoc Scripts on a Cluster
109
- - [ ] Cluster Metrics
110
- - [ ] Add a Node to a Cluster
111
- - [ ] Replace a Node in a Cluster
112
- - [ ] Remove a Node from a Cluster
113
- - [ ] Take an HBase Snapshot
114
- - [ ] View an HBase Snapshot Schedule
115
- - [ ] Restore HBase Tables
116
- - [ ] Update an HBase Snapshot Schedule
99
+ - [x] Cluster API
100
+ - [x] List All Clusters
101
+ - [x] Create a New Cluster
102
+ - [x] View Cluster Configuration
103
+ - [x] Edit Cluster Configuration
104
+ - [x] Clone a Cluster
105
+ - [x] Start or Terminate a Cluster
106
+ - [x] Check Cluster Status
107
+ - [x] Delete a Cluster
108
+ - [x] Reassign Cluster Label
109
+ - [x] Run Adhoc Scripts on a Cluster
110
+ - [x] Cluster Metrics
111
+ - [x] Add a Node to a Cluster
112
+ - [x] Replace a Node in a Cluster
113
+ - [x] Remove a Node from a Cluster
114
+ - [x] Take an HBase Snapshot
115
+ - [x] View an HBase Snapshot Schedule
116
+ - [x] Restore HBase Tables
117
+ - [x] Update an HBase Snapshot Schedule
117
118
 
118
119
 
119
120
  ## Contributing
@@ -1,6 +1,7 @@
1
1
  require "qubole/version"
2
2
  require "qubole/exceptions"
3
3
  require "qubole/commands"
4
+ require "qubole/cluster"
4
5
  require "net/http"
5
6
  require "json"
6
7
 
@@ -0,0 +1,134 @@
1
+ module Qubole
2
+
3
+ # To look for what all parameters need to be passed go to http://docs.qubole.com/en/latest/rest-api/cluster_api/
4
+ # Pass the params as hash from above doc link
5
+ class Cluster
6
+
7
+ # List existing clusters present in your account.
8
+ def self.list
9
+ Qubole.get("/clusters")
10
+ end
11
+
12
+ # Show information about the cluster with id/label `cluster_id_label`.
13
+ def self.show(cluster_id_label)
14
+ Qubole.get("/clusters/#{cluster_id_label}")
15
+ end
16
+
17
+ # Create a new cluster
18
+ # http://docs.qubole.com/en/latest/rest-api/cluster_api/create-new-cluster.html
19
+ def self.create(params={})
20
+ Qubole.post("/clusters", params)
21
+ end
22
+
23
+ # Update the cluster with id/label `cluster_id_label` using information provided in `cluster_info`.
24
+ # Assign id/label of cluster to id
25
+ def self.update(id, params={})
26
+ Qubole.put("/clusters/#{id}", params)
27
+ end
28
+
29
+ # Clone the cluster with id/label `cluster_id_label` using information provided in `cluster_info`.
30
+ # Assign id/label of cluster to id
31
+ def self.clone(id, params={})
32
+ Qubole.post("/clusters/#{id}/clone", params)
33
+ end
34
+
35
+ # Start the cluster with id/label.
36
+ def self.start(id)
37
+ params = {}
38
+ params[:state] = "start"
39
+ Qubole.put("/clusters/#{id}/state", {:state => "start"})
40
+ end
41
+
42
+ # Show the status of the cluster with id/label.
43
+ def self.status(id)
44
+ Qubole.get("/clusters/#{id}/state")
45
+ end
46
+
47
+ # Terminate the cluster with id/label `cluster_id_label`.
48
+ def self.terminate(id)
49
+ Qubole.put("/clusters/#{id}/state", {:state => "terminate"})
50
+ end
51
+
52
+ # Delete the cluster with id/label.
53
+ def self.delete(id)
54
+ Qubole.delete("/clusters/#{id}")
55
+ end
56
+
57
+ # Reassign a label from one cluster to another.
58
+ # http://docs.qubole.com/en/latest/rest-api/cluster_api/reassign-cluster-label.html#parameters
59
+ def self.reassign_label(params={})
60
+ Qubole.put("/clusters/reassign-label", params)
61
+ end
62
+
63
+ # Run an adhoc script
64
+ # http://docs.qubole.com/en/latest/rest-api/cluster_api/run-adhoc-scripts-cluster.html
65
+ # Assign id/label of cluster to id.
66
+ def self.run_script(id, params={})
67
+ Qubole.put("/clusters/#{id}/runscript.json", params)
68
+ end
69
+
70
+ # To look for cluster metrics
71
+ # http://docs.qubole.com/en/latest/rest-api/cluster_api/metrics-cluster.html
72
+ # Assign id/label of cluster to id.
73
+ def self.metrics(id, params={})
74
+ Qubole.get("/clusters/#{id}/metrics", params)
75
+ end
76
+
77
+ # To change the minimum and maximum size of a running cluster
78
+ # http://docs.qubole.com/en/latest/rest-api/cluster_api/resize-cluster.html
79
+ # Assign id/label of cluster to id.
80
+ def self.resize(id, params={})
81
+ Qubole.put("/clusters/#{id}",params)
82
+ end
83
+
84
+ # Add a node to an existing cluster
85
+ # http://docs.qubole.com/en/latest/rest-api/cluster_api/add-node.html
86
+ # Assign id/label of cluster to id.
87
+ def self.add_node(id, params={})
88
+ Qubole.post("/clusters/#{id}/nodes", params)
89
+ end
90
+
91
+ # Replaces a slave node in a cluster
92
+ # http://docs.qubole.com/en/latest/rest-api/cluster_api/replace-node.html
93
+ # Assign id/label of cluster to id.
94
+ def self.replace_node(id, params={})
95
+ params["command"] = "replace"
96
+ Qubole.put("/clusters/#{id}/nodes", params)
97
+ end
98
+
99
+ # Removes a slave node from a cluster
100
+ # http://docs.qubole.com/en/latest/rest-api/cluster_api/remove-node.html#parameters
101
+ # Assign id/label of cluster to id
102
+ def self.remove_node(id, params={})
103
+ Qubole.delete("/clusters/#{id}/nodes", params)
104
+ end
105
+
106
+ # Create hbase snapshot full/incremental
107
+ # http://docs.qubole.com/en/latest/rest-api/cluster_api/hbase-snapshot.html#parameters
108
+ # Assign id/label of cluster to id
109
+ def self.snapshot(id, params={})
110
+ Qubole.post("/clusters/#{id}/snapshots", params)
111
+ end
112
+
113
+ # Get details for snapshot schedule
114
+ def self.get_snapshot_schedule(id)
115
+ Qubole.get("/clusters/#{id}/snapshot_schedule")
116
+ end
117
+
118
+ # Restoring cluster from a given hbase snapshot id
119
+ # http://docs.qubole.com/en/latest/rest-api/cluster_api/restore-point.html#parameters
120
+ # Assign id/label of cluster to id
121
+ def self.restore_point(id, params={})
122
+ Qubole.post("/clusters/#{id}/restore_point", params)
123
+ end
124
+
125
+ # Update for snapshot schedule
126
+ # http://docs.qubole.com/en/latest/rest-api/cluster_api/update-snapshot-schedule.html#parameters
127
+ # Assign id/label of cluster to id
128
+ def self.update_snapshot_schedule(id, params={})
129
+ Qubole.put("/clusters/#{id}/snapshot_schedule", params)
130
+ end
131
+
132
+ end
133
+
134
+ end
@@ -1,3 +1,3 @@
1
1
  module Qubole
2
- VERSION = "0.0.1"
2
+ VERSION = "0.0.2"
3
3
  end
@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
10
10
  spec.email = ["vladnik@demandbase.com"]
11
11
  spec.summary = %q{Qubole REST API client}
12
12
  spec.description = %q{Communication with Qubole REST API in a Ruby way}
13
- spec.homepage = "https://github.com/Demandbase/qubole"
13
+ spec.homepage = "https://github.com/Demandbase/qubole-ruby"
14
14
  spec.license = "MIT"
15
15
 
16
16
  spec.files = `git ls-files -z`.split("\x0")
@@ -0,0 +1,218 @@
1
+ module Qubole
2
+
3
+ describe Cluster do
4
+
5
+ before(:each) do
6
+ Qubole.stub(:http) do |name, path, param|
7
+ @request_type = name
8
+ @url_path = path
9
+ @p = param
10
+ end
11
+ end
12
+
13
+ describe "Cluster list" do
14
+ it "should return correct http arguments" do
15
+ Cluster.list
16
+ expect(@request_type).to eq("GET")
17
+ expect(@url_path).to eq("/clusters")
18
+ expect(@p).to eq({})
19
+ end
20
+ end
21
+
22
+ describe "Show cluster details" do
23
+ it "should return correct http arguments" do
24
+ Cluster.show(1)
25
+ expect(@request_type).to eq("GET")
26
+ expect(@url_path).to eq("/clusters/1")
27
+ expect(@p).to eq({})
28
+ end
29
+ end
30
+
31
+ describe "Create cluster" do
32
+ it "should return correct http arguments" do
33
+ Cluster.create({"cluster"=>{:label => ["my_cluster"], "node_configuration" => {"slave_request_type"=>"spot", "initial_nodes"=>1,
34
+ "spot_instance_settings"=>{"timeout_for_request"=>10, "maximum_spot_instance_percentage"=>60,
35
+ "maximum_bid_price_percentage"=>"100.0"}, "max_nodes"=>10, "master_instance_type"=>"m1.large", "slave_instance_type"=>
36
+ "m1.xlarge"}, "hadoop_settings"=>{"use_hadoop2"=>false, "max_nodes"=>10, "fairscheduler_settings"=>{"default_pool"=>nil},
37
+ "custom_config"=>nil}, "enable_ganglia_monitoring"=>false, "state"=>"DOWN", "node_bootstrap_file"=>"node_bootstrap.sh",
38
+ "use_hadoop2"=>false, "security_settings"=>{"encrypted_ephemerals"=>false}, "ec2_settings"=>{"compute_validated"=>true,
39
+ "aws_region"=>"us-west-2", "aws_preferred_availability_zone"=>"Any", "compute_secret_key"=>"xxxx",
40
+ "vpc_id"=>nil, "compute_access_key"=>"xxxx", "subnet_id"=>nil}, "presto_settings"=>
41
+ {"enable_presto"=>false, "custom_config"=>nil}, "disallow_cluster_termination"=>false}})
42
+
43
+ expect(@request_type).to eq("POST")
44
+ expect(@url_path).to eq("/clusters")
45
+ expect(@p).to eq({"cluster"=>{:label => ["my_cluster"], "node_configuration" => {"slave_request_type"=>"spot", "initial_nodes"=>1,
46
+ "spot_instance_settings"=>{"timeout_for_request"=>10, "maximum_spot_instance_percentage"=>60,
47
+ "maximum_bid_price_percentage"=>"100.0"}, "max_nodes"=>10, "master_instance_type"=>"m1.large", "slave_instance_type"=>
48
+ "m1.xlarge"}, "hadoop_settings"=>{"use_hadoop2"=>false, "max_nodes"=>10, "fairscheduler_settings"=>{"default_pool"=>nil},
49
+ "custom_config"=>nil}, "enable_ganglia_monitoring"=>false, "state"=>"DOWN", "node_bootstrap_file"=>"node_bootstrap.sh",
50
+ "use_hadoop2"=>false, "security_settings"=>{"encrypted_ephemerals"=>false}, "ec2_settings"=>{"compute_validated"=>true, "aws_region"=>"us-west-2", "aws_preferred_availability_zone"=>"Any", "compute_secret_key"=>"xxxx",
51
+ "vpc_id"=>nil, "compute_access_key"=>"xxxx", "subnet_id"=>nil}, "presto_settings"=>
52
+ {"enable_presto"=>false, "custom_config"=>nil}, "disallow_cluster_termination"=>false}})
53
+
54
+ end
55
+ end
56
+
57
+
58
+ describe "Update cluster" do
59
+ it "label should return correct http arguments" do
60
+ Cluster.update(1, {'cluster' => { 'label' => ['test_label']}})
61
+ expect(@request_type).to eq("PUT")
62
+ expect(@url_path).to eq("/clusters/1")
63
+ expect(@p).to eq({'cluster' => { 'label' => ['test_label']}})
64
+ end
65
+ end
66
+
67
+ describe "Clone cluster" do
68
+ it "with label should return correct http arguments" do
69
+ Cluster.clone(1, {"cluster" => {"label" => ["test_label1", "test_label2"]}})
70
+ expect(@request_type).to eq("POST")
71
+ expect(@url_path).to eq("/clusters/1/clone")
72
+ expect(@p).to eq({"cluster" => {"label" => ["test_label1", "test_label2"]}})
73
+ end
74
+ end
75
+
76
+ describe "Start Cluster" do
77
+ it "should return correct http arguments" do
78
+ Cluster.start(1)
79
+ expect(@request_type).to eq("PUT")
80
+ expect(@url_path).to eq("/clusters/1/state")
81
+ expect(@p).to eq({:state => "start"})
82
+ end
83
+ end
84
+
85
+ describe "Status of cluster" do
86
+ it "should return correct http arguments" do
87
+ Cluster.status(1)
88
+ expect(@request_type).to eq("GET")
89
+ expect(@url_path).to eq("/clusters/1/state")
90
+ expect(@p).to eq({})
91
+ end
92
+ end
93
+
94
+ describe "Terminate Cluster" do
95
+ it "should return correct http arguments" do
96
+ Cluster.terminate(1)
97
+ expect(@request_type).to eq("PUT")
98
+ expect(@url_path).to eq("/clusters/1/state")
99
+ expect(@p).to eq({:state => "terminate"})
100
+ end
101
+ end
102
+
103
+ describe "Delete cluster" do
104
+ it "should return correct http arguments" do
105
+ Cluster.delete(1)
106
+ expect(@request_type).to eq("DELETE")
107
+ expect(@url_path).to eq("/clusters/1")
108
+ expect(@p).to eq({})
109
+ end
110
+ end
111
+
112
+ describe "Reassign label" do
113
+ it "should return correct http arguments" do
114
+ Cluster.reassign_label({'destination_cluster' => '123', 'label' => 'test_label'})
115
+ expect(@request_type).to eq("PUT")
116
+ expect(@url_path).to eq("/clusters/reassign-label")
117
+ expect(@p).to eq({'destination_cluster' => '123', 'label' => 'test_label'})
118
+ end
119
+ end
120
+
121
+ describe "Run script" do
122
+ it "using s3 path should return correct http arguments" do
123
+ Cluster.run_script(1, {"script" => "s3_path"})
124
+ expect(@request_type).to eq("PUT")
125
+ expect(@url_path).to eq("/clusters/1/runscript.json")
126
+ expect(@p).to eq({"script" => "s3_path"})
127
+ end
128
+ end
129
+
130
+ describe "Cluster metrics" do
131
+ it "should return correct http arguments" do
132
+ Cluster.metrics(1, {:metric => "mapred.jobtracker.maps_launched", :hostname => "master", :interval => "hour"})
133
+ expect(@request_type).to eq("GET")
134
+ expect(@url_path).to eq("/clusters/1/metrics")
135
+ expect(@p).to eq({:metric => "mapred.jobtracker.maps_launched", :hostname => "master", :interval => "hour"})
136
+ end
137
+ end
138
+
139
+ describe "Resize cluster" do
140
+ it "should return correct http arguments" do
141
+ Cluster.resize(1, {"node_configuration" => {"initial_nodes" => "3"}, "push" => "true"})
142
+ expect(@request_type).to eq("PUT")
143
+ expect(@url_path).to eq("/clusters/1")
144
+ expect(@p).to eq({"node_configuration" => {"initial_nodes" => "3"}, "push" => "true"})
145
+ end
146
+ end
147
+
148
+ describe "Add node to cluster" do
149
+ it "should return correct http arguments" do
150
+ Cluster.add_node(1, {"node_count" => "2"})
151
+ expect(@request_type).to eq("POST")
152
+ expect(@url_path).to eq("/clusters/1/nodes")
153
+ expect(@p).to eq({"node_count" => "2"})
154
+ end
155
+ end
156
+
157
+ describe "Replace node" do
158
+ it "should return correct http arguments" do
159
+ Cluster.replace_node(1, { "private_dns" => "ip-address"})
160
+ expect(@request_type).to eq("PUT")
161
+ expect(@url_path).to eq("/clusters/1/nodes")
162
+ expect(@p).to eq({ "private_dns" => "ip-address", "command" => "replace"})
163
+ end
164
+ end
165
+
166
+ describe "Remove node" do
167
+ it "should return correct http arguments" do
168
+ Cluster.remove_node(1, {"private_dns" => "ip-address"})
169
+ expect(@request_type).to eq("DELETE")
170
+ expect(@url_path).to eq("/clusters/1/nodes")
171
+ expect(@p).to eq({"private_dns" => "ip-address"})
172
+ end
173
+ end
174
+
175
+ describe "Create Hbase Snapshot" do
176
+ it "with no back up type should return correct http arguments" do
177
+ Cluster.snapshot(1, {"s3_location" => "s3-path"})
178
+ expect(@request_type).to eq("POST")
179
+ expect(@url_path).to eq("/clusters/1/snapshots")
180
+ expect(@p).to eq({"s3_location" => "s3-path"})
181
+ end
182
+ end
183
+
184
+ describe "Get snapshot schedule" do
185
+ it "should return correct http arguments" do
186
+ Cluster.get_snapshot_schedule(1)
187
+ expect(@request_type).to eq("GET")
188
+ expect(@url_path).to eq("/clusters/1/snapshot_schedule")
189
+ expect(@p).to eq({})
190
+ end
191
+ end
192
+
193
+ describe "Restore point" do
194
+ it "should return correct http arguments" do
195
+ Cluster.restore_point(1, { "table_names" => "test", "backup_id" => "abcd", "automatic" => true,
196
+ "overwrite" => true, "s3_location" => "s3-path" })
197
+ expect(@request_type).to eq("POST")
198
+ expect(@url_path).to eq("/clusters/1/restore_point")
199
+ expect(@p).to eq({ "table_names" => "test", "backup_id" => "abcd", "automatic" => true,
200
+ "overwrite" => true, "s3_location" => "s3-path" })
201
+
202
+ end
203
+ end
204
+
205
+ describe "Update snapshot schedule" do
206
+ it "should return correct http arguments" do
207
+ Cluster.update_snapshot_schedule(1, {"snapshot-schedule" => { "frequency_unit" => "hours",
208
+ "frequency_num" => 20, "s3_location" => "s3-path"}})
209
+ expect(@request_type).to eq("PUT")
210
+ expect(@url_path).to eq("/clusters/1/snapshot_schedule")
211
+ expect(@p).to eq({"snapshot-schedule" => { "frequency_unit" => "hours",
212
+ "frequency_num" => 20, "s3_location" => "s3-path"}})
213
+
214
+ end
215
+ end
216
+
217
+ end
218
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: qubole
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Volodymyr Ladnik
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-07-07 00:00:00.000000000 Z
11
+ date: 2016-10-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -39,6 +39,7 @@ files:
39
39
  - README.md
40
40
  - Rakefile
41
41
  - lib/qubole.rb
42
+ - lib/qubole/cluster.rb
42
43
  - lib/qubole/command.rb
43
44
  - lib/qubole/commands.rb
44
45
  - lib/qubole/commands/composite.rb
@@ -54,6 +55,7 @@ files:
54
55
  - lib/qubole/exceptions.rb
55
56
  - lib/qubole/version.rb
56
57
  - qubole.gemspec
58
+ - spec/qubole/cluster_spec.rb
57
59
  - spec/qubole/command_spec.rb
58
60
  - spec/qubole/commands/composite_spec.rb
59
61
  - spec/qubole/commands/db_export_spec.rb
@@ -68,7 +70,7 @@ files:
68
70
  - spec/qubole/exceptions_spec.rb
69
71
  - spec/qubole_spec.rb
70
72
  - spec/spec_helper.rb
71
- homepage: https://github.com/Demandbase/qubole
73
+ homepage: https://github.com/Demandbase/qubole-ruby
72
74
  licenses:
73
75
  - MIT
74
76
  metadata: {}
@@ -93,6 +95,7 @@ signing_key:
93
95
  specification_version: 4
94
96
  summary: Qubole REST API client
95
97
  test_files:
98
+ - spec/qubole/cluster_spec.rb
96
99
  - spec/qubole/command_spec.rb
97
100
  - spec/qubole/commands/composite_spec.rb
98
101
  - spec/qubole/commands/db_export_spec.rb