haas 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6e8a464110c902c14ef830dc3a2171248f3aa258
-  data.tar.gz: 760ca88c7ee422e94b7be779b258d142f1381730
+  metadata.gz: ee3233c4653204b2dd02276bb560df79ee96bb79
+  data.tar.gz: 70f99df4a7b12194e9385913b6df379485ce20f1
 SHA512:
-  metadata.gz: 35e900d38e69ca79d7a20c66e27fb2d598b6abe001396bbaa29b85eea601f1165012bcfd10d340f8bc96a2d344480e712e049350af71393d9c0e883bc3269864
-  data.tar.gz: 671575fd90a0a218452f868e580d2f164364eb8baa53ce156b3ce07d4d1ad521faf8e9215f438dd8f4926318adfbcce432994bb51acd30d7b31eeffb2167cf24
+  metadata.gz: 6a93748184eb48c0f7dc2d9f5d1aba8517e6b3a824db71697ea9a653ee374b066d4dc3e5fa6a7fc5be8d388412a8a51311572b6f00eb14ca6063f0273617da0e
+  data.tar.gz: a12e34aa46fc992f8030b20504e4e18a554fda6de4156809abe484d5282bf63eb8dc2a499f7cc747a5251e348fc0c90d1893a3b086d36b870be4c57b664510b9

data/lib/haas.rb

@@ -13,7 +13,8 @@ class Haas
     Haas::Aws.connect
     @cluster=Haas::Cluster.create(
       :aws_region => Haas::Aws.region,
-      :ssh_user => "root"
+      :ssh_user => Haas::Aws.ssh_user,
+      :distro => "ubuntu12"
     )
     if Haas::Aws.nb_instance_available >= Haas::Config.options[:nb_instances].to_i
       Haas::Aws.create_key_pair

data/lib/haas/aws.rb

@@ -4,6 +4,10 @@ class Haas
   class Aws
     CENTOS_IMAGES = {
       "6.5" => {
+        "config" => {
+          "ssh_user" => "root",
+          "root_device" => "/dev/sda"
+        },
         "us-east-1"=>"ami-8997afe0",
         "us-west-2"=>"ami-b6bdde86",
         "us-west-1"=>"ami-1a013c5f",
@@ -14,6 +18,10 @@ class Haas
         "sa-east-1"=>"ami-7d02a260"
       },
       "7" => {
+        "config" => {
+          "ssh_user" => "root",
+          "root_device" => "/dev/sda"
+        },
         "us-east-1"=>"ami-96a818fe",
         "us-west-2"=>"ami-c7d092f7",
         "us-west-1"=>"ami-6bcfc42e",
@@ -25,6 +33,24 @@ class Haas
       }
     }
 
+    UBUNTU_IMAGES = {
+      "12.04" => {
+        "config" => {
+          "ssh_user" => "ubuntu",
+          "root_device" => "/dev/sda1"
+        },
+        "ap-northeast-1" => "ami-f96b40f8",
+        "ap-southeast-1" => "ami-da1e3988",
+        "eu-central-1" => "ami-643c0a79",
+        "eu-west-1" => "ami-6ca1011b",
+        "sa-east-1" => "ami-11d4610c",
+        "us-east-1" => "ami-34cc7a5c",
+        "us-west-1" => "ami-b7515af2",
+        "ap-southeast-2" => "ami-9f0e6ca5",
+        "us-west-2" => "ami-0f47053f"
+      }
+    }
+
     def self.connect
       @region = Haas::Config.options[:aws_region] || 'us-east-1'
       AWS.config(
@@ -43,6 +69,10 @@ class Haas
       @region
     end
 
+    def self.ssh_user
+      UBUNTU_IMAGES["12.04"]["config"]["ssh_user"]
+    end
+
     def self.nb_instance_available
       account_attributes = ec2.client.describe_account_attributes\
       .data[:account_attribute_set]\
@@ -65,7 +95,7 @@ class Haas
     end
 
     def self.launch_instances
-      image_id = CENTOS_IMAGES["6.5"][region]
+      image_id = UBUNTU_IMAGES["12.04"][region]
 
       if !ec2.security_groups.filter('group-name', 'haas-security-group').first
         security_group = ec2.security_groups.create('haas-security-group')
@@ -85,7 +115,7 @@ class Haas
         :security_groups => ['haas-security-group'],
         :block_device_mappings => [
           {
-            :device_name => "/dev/sda",
+            :device_name => "/dev/sda1",
             :ebs => {
               :volume_size => 8, # 8 GiB
               :delete_on_termination => true

data/lib/haas/blueprints.rb

@@ -28,17 +28,22 @@ class Haas
 
     def self.get_blueprint
       {
+        "configurations" => [
+          {
+            "hive-site"=> {
+              "javax.jdo.option.ConnectionPassword"=> "hive"
+            }
+          },
+          {
+            "nagios-env" => {
+               "nagios_contact" => "admin@localhost"
+            }
+          }
+        ],
         "host_groups" => [
           {
             "name" => "master",
-            "configurations" => [
-              {
-                "global" => {
-                  "nagios_contact" => "me@my-awesome-domain.example"
-                }
-              }
-            ],
-              "components" => [
+            "components" => [
               {
                 "name" => "NAMENODE"
               },
@@ -62,6 +67,9 @@ class Haas
               },
               {
                 "name" => "GANGLIA_MONITOR"
+              },
+              {
+                "name" => "APP_TIMELINE_SERVER"
               }
             ],
             "cardinality" => "1"
@@ -96,7 +104,7 @@ class Haas
         ],
        "Blueprints" => {
           "stack_name" => "HDP",
-          "stack_version" => "2.1"
+          "stack_version" => "2.2"
         }
       }
     end

data/lib/haas/chef.rb

@@ -6,7 +6,10 @@ class Haas
     def self.setup_cluster
       install_chef_server
       write_knife_config_file
-      cookbooks=[{'name' => 'ambari','url' => "https://supermarket.getchef.com/cookbooks/ambari/download" }]
+      cookbooks=[
+        {'name' => 'ambari', 'url' => "https://supermarket.getchef.com/cookbooks/ambari/download" },
+        {'name' => 'apt', 'url' => "https://supermarket.getchef.com/cookbooks/apt/download" }
+      ]
       cookbooks.each do |cb|
         download_cookbook cb['name'], cb['url']
       end
@@ -19,16 +22,20 @@ class Haas
       threads.each { |thr| thr.join }
     end
 
+    def self.chef_install_cmd
+      install_cmd = {
+        'centos6'    => 'curl https://packagecloud.io/install/repositories/chef/stable/script.rpm | sudo bash && sudo yum install chef-server-core-12.0.0-1.el6.x86_6',
+        'ubuntu12' => 'curl https://packagecloud.io/install/repositories/chef/stable/script.deb | sudo bash && sudo apt-get install chef-server-core=12.0.0-1'
+      }
+      install_cmd[Haas.cluster.distro]
+    end
+
     def self.install_chef_server
       require 'net/ssh'
       chef_server = Haas.cluster.get_chef_server
-      user = Haas.cluster.ssh_user
-      chef_server_file = "chef-server-core-12.0.0_rc.5-1.el5.x86_64.rpm"
-      chef_server_url = "https://packagecloud.io/chef/stable/download?distro=6&filename=#{chef_server_file}"
-      chef_server_local_path = "/tmp/#{chef_server_file}"
 
       Net::SSH.start(
-        chef_server.public_dns_name, user,
+        chef_server.public_dns_name, Haas.cluster.ssh_user,
         :host_key => "ssh-rsa",
         :encryption => "blowfish-cbc",
         :keys => [ Haas.cluster.identity_file_path ],
@@ -36,26 +43,21 @@ class Haas
       ) do |ssh|
         puts "Entering chef server installation on the node #{chef_server.public_dns_name}. This may take a while."
         puts "Disable iptables"
-        ssh.exec!("service iptables stop")
+        ssh.exec!("sudo service iptables stop")
         puts "Downloading and installing the chef server."
-        ssh.exec!(%{
-          until curl -L '#{chef_server_url}' -o #{chef_server_local_path} && rpm -ivh #{chef_server_local_path}; do
-            echo "installing chef server";
-          done
-        })
-        ssh.exec!("rpm -ivh #{chef_server_local_path}")
+        ssh.exec!(self.chef_install_cmd)
         puts "Configuring chef server."
-        ssh.exec!("mkdir -p /etc/opscode/")
-        ssh.exec!(%{echo "nginx['non_ssl_port'] = false" >> /etc/opscode/chef-server.rb})
-        ssh.exec!("chef-server-ctl reconfigure")
+        ssh.exec!("sudo mkdir -p /etc/opscode/")
+        ssh.exec!(%{sudo bash -c "echo \\"nginx['non_ssl_port'] = false\\" >> /etc/opscode/chef-server.rb"})
+        ssh.exec!("sudo chef-server-ctl reconfigure")
 
         client_key = ""
         while !client_key.include?("BEGIN RSA PRIVATE KEY") do
-          client_key = ssh.exec!("chef-server-ctl user-create haas-api HAAS Api haas@ossom.io abc123")
+          client_key = ssh.exec!("sudo chef-server-ctl user-create haas-api HAAS Api haas@ossom.io abc123")
         end
         File.write(Haas.cluster.chef_client_pem_path, client_key)
 
-        org_validator_key = ssh.exec!("chef-server-ctl org-create haas Hadoop as a Service --association_user haas-api")
+        org_validator_key = ssh.exec!("sudo chef-server-ctl org-create haas Hadoop as a Service --association_user haas-api")
         File.write(Haas.cluster.chef_validator_pem_path, org_validator_key)
       end
     end
@@ -90,13 +92,12 @@ class Haas
 
       puts "Bootstrapping node #{node.public_dns_name}"
 
-      user = Haas.cluster.ssh_user
       run_list = ["recipe[ambari::agent]"]
       run_list << "recipe[ambari::server]" if node.ambari_server
 
       Chef::Config.from_file(Haas.cluster.knife_config_path)
       kb = Chef::Knife::Bootstrap.new
-      kb.config[:ssh_user] = user
+      kb.config[:ssh_user] = Haas.cluster.ssh_user
       kb.config[:run_list] = run_list
       kb.config[:use_sudo] = true
       kb.config[:identity_file] = Haas.cluster.identity_file_path

data/lib/haas/config.rb

@@ -36,6 +36,7 @@ class Haas
           table.column :name, :string
           table.column :aws_region, :string
           table.column :ssh_user, :string
+          table.column :distro, :string
         end
         add_index :clusters, :name, unique: true
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: haas
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Julien Pellet
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-22 00:00:00.000000000 Z
+date: 2014-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.1'
+        version: 4.1.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.1'
+        version: 4.1.8
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement