cloudbox-server 0.0.1

data/lib/receiver/rest_api_connector.rb

@@ -0,0 +1,275 @@
+# This file is part of cloudbox-server project
+# @author tnoguer (INGENSI)
+
+require 'uri'
+require 'xmlsimple'
+require 'net/http'
+require 'net/http/digest_auth'
+require 'common/cloudbox_exceptions'
+require 'receiver/connector'
+
+module Receiver
+  # Allow to connects to an API and execute Rest command (GET/POST/PUT/DELETE)
+  # @author mbretaud
+  class RestApiConnector < Connector
+
+    public
+    # The destination IP.
+    attr_reader :dest_ip
+    # The destination port.
+    attr_reader :dest_port
+    # The timeout.
+    attr_reader :timeout
+    # The type of authentification
+    attr_reader :auth_type
+
+    private
+    # The user, used for authentication.
+    @user
+    # The password, used for authentication.
+    @password
+    # The header, used for http requests
+    @header
+    # The http object.
+    @http
+
+    public
+    # Default constructor.
+    # @param dest_ip [String] The destination IP.
+    # @param dest_port [Integer] The destination port.
+    # @param auth_type [String] The type of authentication (BASIC, DIGEST)
+    # @param user [String] The username.
+    # @param password [String] The password.
+    # @raise [ArgumentError] If the value of the arguments are nil or invalid.
+    # @author tnoguer
+    def initialize(dest_ip, dest_port, auth_type, user, password, timeout = 10000, logger = nil)
+      super(logger)
+      @logger.info("Receiver::RestApiConnector   initialize the parameters...")
+      if dest_ip == nil || dest_port == nil || user == nil || password == nil
+        raise ArgumentError.new("Arguments can\'t be nil!")
+      elsif auth_type != "BASIC" && auth_type != "DIGEST"
+        raise ArgumentError.new("Authentication type invalid: " + auth_type)
+      end
+
+      @dest_ip = dest_ip
+      @dest_port = dest_port
+      @auth_type = auth_type
+      @user = user
+      @password = password
+      @timeout = timeout
+      @header = {
+          "Accept" => "application/json",
+          "Content-Type" => "application/json"
+      }
+      @body = ''
+      @http = Net::HTTP.new(@dest_ip, @dest_port)
+
+      @logger.info("Receiver::RestApiConnector   initialize the parameters...")
+    end
+
+    public
+    # Launch an http request to the rest api.
+    # @param request_type The type of request. (GET, POST, PUT, DELETE)
+    # @param address The url address of the request. (ex: /clusters)
+    # @param body The body of the request. (ex: "\{\"Clusters\": \{\"version\" : \"HDP-1.3.0\"\}\}"
+    # @raise [ArgumentError] If the value of the arguments are nil or invalid.
+    # @raise [HTTPError] If the request failed.
+    # @author tnoguer
+    def launch_request(request_type, address, body = '')
+      @logger.info("Receiver::RestApiConnector   launch a #{request_type} request on the '#{address}' with body '#{body}'...")
+
+      uri = URI.parse("http://#{@dest_ip}:#{@dest_port}#{address.strip}")
+      uri.user=@user.strip
+      uri.password=@password.strip
+      response=nil
+      error = false
+
+      if request_type == nil || address == nil || body == nil
+        raise ArgumentError.new("Arguments can\'t be nil!")
+      end
+
+      if @auth_type == "BASIC"
+        case request_type
+          when "GET"
+            request = Net::HTTP::Get.new(address)
+          when "POST"
+            request = Net::HTTP::Post.new(address)
+          when "PUT"
+            request = Net::HTTP::Put.new(address)
+          when "DELETE"
+            request = Net::HTTP::Delete.new(address)
+          else
+            raise ArgumentError.new("Invalid type: " + request_type)
+        end
+
+        request.basic_auth(@user, @password)
+
+        request.body = body
+        response = @http.request(request)
+
+        error = false
+        case request_type
+          when "GET" || "DELETE"
+            if response.code != '200'
+              error = true
+            end
+          when "POST"
+            if response.code != '201'
+              error = true
+            end
+          when "PUT"
+            if response.code != '200' && response.code != '202'
+              error = true
+            end
+        end
+      elsif @auth_type == "DIGEST"
+        Net::HTTP.start(uri.host, uri.port) {|http|
+          http.read_timeout = @timeout
+          case request_type
+            when "GET"
+              request = Net::HTTP::Get.new(uri.request_uri,@header)
+            when "POST"
+              request = Net::HTTP::Post.new(uri.request_uri)
+            when "PUT"
+              request = Net::HTTP::Put.new(uri.request_uri,@header)
+            when "DELETE"
+              request = Net::HTTP::Delete.new(uri.request_uri,@header)
+            else
+              raise ArgumentError.new("Invalid type: " + request_type)
+          end
+
+          request.body = body
+          response = http.request request
+
+          if response['www-authenticate']
+
+            begin
+              auth = Net::HTTP::DigestAuth.new.auth_header(uri, response['www-authenticate'], request_type)
+              request.add_field 'Authorization', auth
+              response = http.request request
+            rescue => e
+              message = get_message_html(response.body)
+            end
+            raise Common::HTTPError.getConsistentError(message, response.code, response.body)
+          end
+
+          error = false
+          case request_type
+            when "GET" || "DELETE"
+              if response.code != '200'
+                error = true
+              end
+            when "POST"
+              if response.code != '302'
+                error = true
+              end
+            when "PUT"
+              if response.code != '200'
+                error = true
+              end
+            else
+              raise ArgumentError.new("Invalid type: " + request_type)
+          end
+
+          #if error && response.body != ""
+          #  begin
+          #    m = JSON.parse(response.body)
+          #    message = m['message']
+          #  rescue
+          #    # @todo raise exception
+          #    message = get_message_html(response.body)
+          #  end
+          #  raise Common::HTTPError.getConsistentError(message, response.code, response.body)
+          #end
+          #return response
+        }
+      end
+
+      if error
+        begin
+          m = JSON.parse(response.body)
+          message = m['message']
+        rescue
+          message = get_message_html(response.body)
+        end
+        raise Common::HTTPError.getConsistentError(message, response.code, response.body)
+      end
+      return response
+    end
+
+    public
+    # Launch an http request to the rest api.
+    # @param request_type The type of request. (GET, POST, PUT, DELETE)
+    # @param address The url address of the request. (ex: /clusters)
+    # @param body The body of the request. (ex: "\{\"Clusters\": \{\"version\" : \"HDP-1.3.0\"\}\}"
+    # @raise [ArgumentError] If the value of the arguments are nil or invalid.
+    # @raise [HTTPError] If the request failed.
+    # @author tnoguer
+    def launch_request_mock(request_type, address, body = nil)
+      uri = URI.parse("http://#{@dest_ip}:#{@dest_port}#{address.strip}")
+      uri.user=@user
+      uri.password=@password
+      response=nil
+
+      if request_type == nil || address == nil || body == nil
+        raise ArgumentError.new("Arguments can\'t be nil!")
+      end
+
+      if @auth_type == "DIGEST"
+        uri = URI.parse("http://#{@dest_ip}:#{@dest_port}#{address.strip}")
+        uri.user=@user.strip
+        uri.password=@password.strip
+        response=nil
+
+        Net::HTTP.start(uri.host, uri.port) {|http|
+          http.read_timeout = @timeout
+          case request_type
+            when "GET"
+              request = Net::HTTP::Get.new(uri.request_uri,@header)
+            when "POST"
+              request = Net::HTTP::Post.new(uri.request_uri,@header)
+            when "PUT"
+              request = Net::HTTP::Put.new(uri.request_uri,@header)
+            when "DELETE"
+              request = Net::HTTP::Delete.new(uri.request_uri,@header)
+            else
+              raise ArgumentError.new("Invalid type: " + request_type)
+          end
+
+          request.body = body
+          response = http.request request
+
+          if response['www-authenticate']
+            begin
+              auth = Net::HTTP::DigestAuth.new.auth_header(uri, response['www-authenticate'], request_type)
+              request.add_field 'Authorization', auth
+              response = http.request request
+
+              return response
+            rescue => e
+              message = get_message_html(response.body)
+            end
+            raise Common::HTTPError.getConsistentError(message, response.code, response.body)
+          end
+        }
+      end
+
+      return response
+    end
+
+    private
+    # Get message for HTML error content.
+    def get_message_html(html)
+      @logger.info("Receiver::RestApiConnector   get the message html '#{html}' of the request...")
+
+      begin
+        parsed_html = XmlSimple.xml_in(html)
+        #$LOG.debug(parsedHTML)
+        return parsed_html['body'][0]['p'][0]['content']
+      rescue
+        return html
+      end
+    end
+
+  end
+end

data/resources/ambari-configurations/HBASE/default_hbase-site

@@ -0,0 +1,33 @@
+{
+  "hbase.hregion.memstore.flush.size": "134217728",
+  "hbase.hstore.compactionThreshold": "3",
+  "hbase.security.authorization": "false",
+  "zookeeper.znode.parent": "/hbase-unsecure",
+  "hbase.regionserver.global.memstore.upperLimit": "0.4",
+  "hbase.security.authentication": "simple",
+  "hbase.rpc.engine": "org.apache.hadoop.hbase.ipc.WritableRpcEngine",
+  "dfs.support.append": "true",
+  "hbase.hregion.majorcompaction": "86400000",
+  "hbase.client.keyvalue.maxsize": "10485760",
+  "hbase.client.scanner.caching": "100",
+  "hbase.cluster.distributed": "true",
+  "hbase.rootdir": "hdfs://__MASTERNODE__:8020/apps/hbase/data",
+  "hbase.regionserver.kerberos.principal": "rs/_HOST@EXAMPLE.COM",
+  "hbase.regionserver.global.memstore.lowerLimit": "0.35",
+  "hbase.regionserver.handler.count": "30",
+  "hbase.tmp.dir": "/var/log/hbase",
+  "hbase.hregion.memstore.mslab.enabled": "true",
+  "hbase.regionserver.optionalcacheflushinterval": "10000",
+  "hbase.hstore.blockingStoreFiles": "7",
+  "hbase.zookeeper.useMulti": "true",
+  "hfile.block.cache.size": "0.25",
+  "hbase.hregion.max.filesize": "1073741824",
+  "hbase.master.lease.thread.wakefrequency": "3000",
+  "hbase.hregion.memstore.block.multiplier": "2",
+  "hbase.master.kerberos.principal": "hm/_HOST@EXAMPLE.COM",
+  "zookeeper.session.timeout": "60000",
+  "hbase.zookeeper.property.clientPort": "2181",
+  "hbase.master.keytab.file": "/etc/security/keytabs/hm.service.keytab",
+  "dfs.client.read.shortcircuit": "true",
+  "hbase.zookeeper.quorum": "__HOST_ZOOKEEPER__"
+}

data/resources/ambari-configurations/HDFS/default_hdfs-site

@@ -0,0 +1,49 @@
+{
+  "dfs.https.address": "__MASTERNODE__:50470",
+  "dfs.http.address": "__MASTERNODE__:50070",
+  "dfs.datanode.max.xcievers": "4096",
+  "dfs.safemode.threshold.pct": "1.0f",
+  "dfs.datanode.http.address": "0.0.0.0:50075",
+  "dfs.datanode.address": "0.0.0.0:50010",
+  "dfs.datanode.failed.volumes.tolerated": "0",
+  "dfs.namenode.keytab.file": "/etc/security/keytabs/nn.service.keytab",
+  "ipc.server.read.threadpool.size": "5",
+  "dfs.datanode.data.dir.perm": "750",
+  "dfs.datanode.du.pct": "0.85f",
+  "ipc.server.max.response.size": "5242880",
+  "dfs.datanode.du.reserved": "1",
+  "dfs.datanode.socket.write.timeout": "0",
+  "dfs.namenode.handler.count": "100",
+  "dfs.web.ugi": "gopher,gopher",
+  "dfs.datanode.keytab.file": "/etc/security/keytabs/dn.service.keytab",
+  "dfs.name.dir": "/hadoop/hdfs/namenode",
+  "dfs.heartbeat.interval": "3",
+  "dfs.balance.bandwidthPerSec": "6250000",
+  "dfs.permissions.supergroup": "hdfs",
+  "dfs.cluster.administrators": " hdfs",
+  "dfs.secondary.namenode.keytab.file": "/etc/security/keytabs/nn.service.keytab",
+  "dfs.hosts.exclude": "/etc/hadoop/dfs.exclude",
+  "dfs.permissions": "true",
+  "dfs.namenode.kerberos.https.principal": "host/_HOST@EXAMPLE.COM",
+  "dfs.datanode.kerberos.principal": "dn/_HOST@EXAMPLE.COM",
+  "dfs.web.authentication.kerberos.keytab": "/etc/security/keytabs/spnego.service.keytab",
+  "dfs.webhdfs.enabled": "false",
+  "dfs.secondary.namenode.kerberos.https.principal": "host/_HOST@EXAMPLE.COM",
+  "dfs.secondary.https.port": "50490",
+  "dfs.block.size": "134217728",
+  "dfs.datanode.ipc.address": "0.0.0.0:8010",
+  "dfs.data.dir": "/hadoop/hdfs/data",
+  "dfs.namenode.check.stale.datanode": "true",
+  "dfs.access.time.precision": "0",
+  "dfs.blockreport.initialDelay": "120",
+  "dfs.https.port": "50470",
+  "dfs.block.access.token.enable": "true",
+  "dfs.umaskmode": "077",
+  "dfs.replication": "3",
+  "dfs.secondary.http.address": "__MASTERNODE__:50090",
+  "dfs.secondary.namenode.kerberos.principal": "nn/_HOST@EXAMPLE.COM",
+  "dfs.replication.max": "50",
+  "dfs.hosts": "/etc/hadoop/dfs.include",
+  "dfs.namenode.kerberos.principal": "nn/_HOST@EXAMPLE.COM",
+  "dfs.block.local-path-access.user": "hbase"
+}

data/resources/ambari-configurations/HIVE/default_hive-site

@@ -0,0 +1,33 @@
+{
+  "hive.mapred.reduce.tasks.speculative.execution": "false",
+  "fs.file.impl.disable.cache": "true",
+  "hive.enforce.sorting": "true",
+  "javax.jdo.option.ConnectionPassword": "admin",
+  "hive.metastore.uris": "thrift://__MASTERNODE__:9083",
+  "hive.optimize.bucketmapjoin.sortedmerge": "true",
+  "hive.optimize.reducededuplication.min.reducer": "1",
+  "hive.metastore.local": "false",
+  "javax.jdo.option.ConnectionURL": "jdbc:mysql://__MASTERNODE__/hive?createDatabaseIfNotExist=true",
+  "hive.semantic.analyzer.factory.impl": "org.apache.hivealog.cli.HCatSemanticAnalyzerFactory",
+  "hive.metastore.client.socket.timeout": "60",
+  "hive.metastore.warehouse.dir": "/apps/hive/warehouse",
+  "hive.server2.enable.doAs": "true",
+  "hive.auto.convert.join.noconditionaltask.size": "1000000000",
+  "hive.auto.convert.sortmerge.join": "true",
+  "javax.jdo.option.ConnectionDriverName": "com.mysql.jdbc.Driver",
+  "hive.map.aggr": "true",
+  "hive.enforce.bucketing": "true",
+  "hadoop.clientside.fs.operations": "true",
+  "hive.metastore.cache.pinobjtypes": "Table,Database,Type,FieldSchema,Order",
+  "hive.auto.convert.sortmerge.join.noconditionaltask": "true",
+  "hive.mapjoin.bucket.cache.size": "10000",
+  "hive.metastore.execute.setugi": "true",
+  "fs.hdfs.impl.disable.cache": "true",
+  "hive.optimize.bucketmapjoin": "true",
+  "hive.security.authorization.enabled": "true",
+  "hive.auto.convert.join.noconditionaltask": "true",
+  "javax.jdo.option.ConnectionUserName": "hive",
+  "hive.auto.convert.join": "true",
+  "hive.security.authorization.manager": "org.apache.hcatalog.security.HdfsAuthorizationProvider",
+  "hive.optimize.mapjoin.mapreduce": "true"
+}

data/resources/ambari-configurations/MAPREDUCE/default_mapred-site

@@ -0,0 +1,72 @@
+{
+  "mapred.reduce.tasks.speculative.execution": "false",
+  "mapred.tasktracker.map.tasks.maximum": "4",
+  "mapred.hosts.exclude": "/etc/hadoop/mapred.exclude",
+  "mapreduce.tasktracker.group": "hadoop",
+  "mapred.job.reduce.input.buffer.percent": "0.0",
+  "mapreduce.reduce.input.limit": "10737418240",
+  "mapred.map.tasks.speculative.execution": "false",
+  "mapreduce.fileoutputcommitter.marksuccessfuljobs": "false",
+  "mapreduce.jobtracker.kerberos.principal": "jt/_HOST@EXAMPLE.COM",
+  "mapred.output.compression.type": "BLOCK",
+  "mapred.userlog.retain.hours": "24",
+  "mapred.job.reuse.jvm.num.tasks": "1",
+  "mapred.system.dir": "/mapred/system",
+  "mapreduce.tasktracker.keytab.file": "/etc/security/keytabs/tt.service.keytab",
+  "mapred.task.tracker.task-controller": "org.apache.hadoop.mapred.DefaultTaskController",
+  "io.sort.factor": "100",
+  "mapreduce.history.server.http.address": "__MASTERNODE__:51111",
+  "mapred.jobtracker.maxtasks.per.job": "-1",
+  "mapred.cluster.reduce.memory.mb": "-1",
+  "io.sort.spill.percent": "0.9",
+  "mapred.reduce.parallel.copies": "30",
+  "tasktracker.http.threads": "50",
+  "mapred.healthChecker.script.path": "file:////mapred/jobstatus",
+  "mapreduce.cluster.administrators": " hadoop",
+  "jetty.connector": "org.mortbay.jetty.nio.SelectChannelConnector",
+  "mapred.inmem.merge.threshold": "1000",
+  "mapred.job.reduce.memory.mb": "-1",
+  "mapred.job.map.memory.mb": "-1",
+  "mapreduce.jobhistory.kerberos.principal": "jt/_HOST@EXAMPLE.COM",
+  "mapred.cluster.map.memory.mb": "-1",
+  "mapred.jobtracker.retirejob.interval": "21600000",
+  "mapred.job.tracker.persist.jobstatus.hours": "1",
+  "mapred.cluster.max.map.memory.mb": "-1",
+  "mapred.reduce.slowstart.completed.maps": "0.05",
+  "hadoop.job.history.user.location": "none",
+  "mapred.job.tracker.handler.count": "50",
+  "mapred.healthChecker.interval": "135000",
+  "mapred.jobtracker.blacklist.fault-bucket-width": "15",
+  "mapred.task.timeout": "600000",
+  "mapred.jobtracker.taskScheduler": "org.apache.hadoop.mapred.CapacityTaskScheduler",
+  "mapred.max.tracker.blacklists": "16",
+  "mapreduce.jobhistory.keytab.file": "/etc/security/keytabs/jt.service.keytab",
+  "mapred.map.output.compression.codec": "org.apache.hadoop.io.compress.SnappyCodec",
+  "mapred.jobtracker.retirejob.check": "10000",
+  "mapred.tasktracker.tasks.sleeptime-before-sigkill": "250",
+  "mapreduce.jobtracker.staging.root.dir": "/user",
+  "mapred.job.shuffle.input.buffer.percent": "0.7",
+  "mapred.jobtracker.completeuserjobs.maximum": "5",
+  "mapred.job.tracker.persist.jobstatus.active": "false",
+  "mapred.tasktracker.reduce.tasks.maximum": "2",
+  "mapreduce.history.server.embedded": "false",
+  "mapred.job.tracker.http.address": "__MASTERNODE__:50030",
+  "mapred.queue.names": "default",
+  "mapred.job.tracker.history.completed.location": "/mapred/history/done",
+  "mapred.child.java.opts": "-server -Xmx768m -Djava.net.preferIPv4Stack=true",
+  "mapred.jobtracker.blacklist.fault-timeout-window": "180",
+  "mapreduce.jobtracker.split.metainfo.maxsize": "50000000",
+  "mapred.healthChecker.script.timeout": "60000",
+  "mapred.jobtracker.restart.recover": "false",
+  "mapreduce.jobtracker.keytab.file": "/etc/security/keytabs/jt.service.keytab",
+  "mapred.hosts": "/etc/hadoop/mapred.include",
+  "mapred.local.dir": "/hadoop/mapred",
+  "mapreduce.tasktracker.kerberos.principal": "tt/_HOST@EXAMPLE.COM",
+  "mapred.job.tracker.persist.jobstatus.dir": "/etc/hadoop/health_check",
+  "mapred.job.tracker": "__MASTERNODE__:50300",
+  "io.sort.record.percent": ".2",
+  "mapred.cluster.max.reduce.memory.mb": "-1",
+  "io.sort.mb": "200",
+  "mapred.job.shuffle.merge.percent": "0.66",
+  "mapred.child.root.logger": "INFO,TLA"
+}