vayacondios-server 0.1.2 → 0.1.6

data/lib/vayacondios/version.rb

@@ -1,3 +1,3 @@
 class Vayacondios
-  VERSION = '0.1.2'
+  VERSION = '0.1.6'
 end

data/pom.xml

@@ -0,0 +1,97 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.infochimps</groupId>
+  <artifactId>vayacondios</artifactId>
+  <packaging>jar</packaging>
+  <version>1.0-SNAPSHOT</version>
+  <name>java-common</name>
+  <url>http://maven.apache.org</url>
+
+  <parent>
+    <groupId>com.infochimps</groupId>
+    <artifactId>parent-pom</artifactId>
+    <version>1.0.0-SNAPSHOT</version>
+  </parent>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <version>2.0</version>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+              <transformers>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                  <mainClass>com.infochimps.vayacondios.VCDIntegrationTest</mainClass>
+                </transformer>
+              </transformers>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+	<groupId>org.codehaus.mojo</groupId>
+	<artifactId>exec-maven-plugin</artifactId>
+	<version>1.2.1</version>
+	<configuration>
+	  <executable>java</executable>
+	  <arguments>
+	    <argument>-cp</argument>
+	    <classpath/>
+	    <argument>-jar</argument>
+	    <argument>target/${project.artifactId}-${project.version}.jar</argument>
+	  </arguments>
+	</configuration>
+      </plugin>
+    </plugins>
+  </build>
+  
+  <repositories>
+    <!-- Infochimps Repositories -->
+    <repository>
+      <id>infochimps.releases</id>
+      <name>Infochimps Internal Repository</name>
+      <url>https://s3.amazonaws.com/artifacts.chimpy.us/maven-s3p/releases</url>
+    </repository>
+    <repository>
+      <id>infochimps.snapshots</id>
+      <name>Infochimps Internal Repository</name>
+      <url>https://s3.amazonaws.com/artifacts.chimpy.us/maven-s3p/snapshots</url>
+      <snapshots>
+	<enabled>true</enabled>
+	<updatePolicy>always</updatePolicy>
+      </snapshots>
+    </repository>
+  </repositories>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.google.code.gson</groupId>
+      <artifactId>gson</artifactId>
+      <version>2.2.2</version>
+    </dependency>
+    <dependency>
+      <groupId>commons-codec</groupId>
+      <artifactId>commons-codec</artifactId>
+      <version>1.2</version>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>1.7.2</version>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>3.8.1</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+</project>

data/scripts/hadoop_monitor/configurable.rb

@@ -1,7 +1,7 @@
 require 'configliere'
 require 'logger'
 
-module Vayacondios
+class Vayacondios
 
   module Configurable
 

data/scripts/hadoop_monitor/hadoop_attempt_scraper.rb

@@ -27,16 +27,19 @@ class HadoopAttemptScraper < Nibbler
 
   def to_attempts
     attempts.map do |attempt|
+      start_time  = Time.parse(attempt.start_time) rescue nil
+      finish_time = attempt.finish_time.length > 0 ? Time.parse(attempt.finish_time) : nil
       {
         _id:         attempt.attempt_id.to_s,
         task_id:     task_id,
         host:        attempt.machine.to_s.gsub(/^http:\/\//, '').gsub(/:[0-9]+$/, ''),
         status:      attempt.status,
         progress:    attempt.progress.to_f / 100.0,
-        start_time:  Time.parse(attempt.start_time),
-        finish_time: attempt.finish_time.length > 0 ? Time.parse(attempt.finish_time) : nil,
+        start_time:  start_time,
+        finish_time: finish_time,
+        duration:    start_time ? (finish_time || Time.now) - start_time : nil,
         errors:      attempt.errors
       }
     end
   end
-end
+end

data/scripts/hadoop_monitor/hadoop_client.rb

@@ -10,7 +10,7 @@ require 'pp'
 require 'gorillib/string/inflections'
 require 'swineherd-fs'
 
-module Vayacondios
+class Vayacondios
 
   class HadoopClient
 
@@ -87,6 +87,11 @@ module Vayacondios
       finished_status = [:FAILED, :KILLED, :COMPLETE]
       failed_status   = [:FAILED]
 
+
+      # not sure what is what. I'm guessing
+      # JobStatus.getStartTime corresponds to the
+      # launch time in the logs
+
       start_time      = Time.at(job_status.get_start_time / 1000)
       reduce_progress = job.reduce_progress
       map_progress    = job.map_progress
@@ -100,22 +105,16 @@ module Vayacondios
         _id:              job_id.to_s,
         name:             job.get_job_name.to_s,
 
-                          # not sure what is what. I'm guessing
-                          # JobStatus.getStartTime corresponds to the
-                          # launch time in the logs, but I'm going to
-                          # go ahead and use it twice here.
-
-        launch_time:      start_time,
-        submit_time:      start_time,
+        start_time:       start_time,
         finish_time:      finish_time,
 
-        run_duration:     run_duration,
+        duration:         run_duration,
 
         map_eta:          map_eta,
         reduce_eta:       reduce_eta,
         eta:              reduce_eta,
 
-        job_status:       case job_status.get_run_state
+        status:       case job_status.get_run_state
                           when JobStatus::FAILED    then :FAILED
                           when JobStatus::KILLED    then :KILLED
                           when JobStatus::PREP      then :PREP
@@ -128,9 +127,7 @@ module Vayacondios
         failed_maps:      num_tasks(job_id, :map,    failed_status),
         failed_reduces:   num_tasks(job_id, :reduce, failed_status),
 
-        counters:         parse_counters(job.get_counters),
-        type:             :job,
-
+        counters:         parse_counters(job.get_counters)
       }
 
       job_event = {
@@ -195,13 +192,17 @@ module Vayacondios
     # object that represents it.
     #
     def parse_task task_report, task_type, parent_job_id
+      start_time  = task_report.get_start_time > 0  ? Time.at(task_report.get_start_time  / 1000) : nil
+      finish_time = task_report.get_finish_time > 0 ? Time.at(task_report.get_finish_time / 1000) : nil
+
       {
         _id:                   task_report.get_task_id.to_s,
-        job_id:                parent_job_id,
-        task_type:             task_type,
-        task_status:           task_report.get_current_status.to_s,
-        start_time:            Time.at(task_report.get_start_time / 1000),
-        finish_time:           task_report.get_finish_time > 0 ? Time.at(task_report.get_finish_time / 1000) : nil,
+        job_id:                parent_job_id.to_s,
+        type:                  task_type,
+        status:                task_report.get_current_status.to_s,
+        start_time:            start_time,
+        finish_time:           finish_time,
+        duration:              start_time ? (finish_time || Time.now) - start_time : nil,
         counters:              parse_counters(task_report.get_counters),
         diagnostics:           task_report.get_diagnostics.map(&:to_s),
         successful_attempt_id: task_report.get_successful_task_attempt.to_s
@@ -212,7 +213,7 @@ module Vayacondios
       {
         t: Time.now,
         d: {
-          job_id:              task_report.get_task_id.to_s,
+          task_id:             task_report.get_task_id.to_s,
           progress:            task_report.get_progress,
           running_attempt_ids: task_report.get_running_task_attempts.map(&:to_s)
         }

data/scripts/hadoop_monitor/hadoop_monitor.rb

@@ -10,7 +10,7 @@ require 'thread'
 require 'open-uri'
 require 'json'
 
-module Vayacondios
+class Vayacondios
 
   class HadoopMonitor
     def initialize
@@ -34,8 +34,8 @@ module Vayacondios
         tasks:    @db.create_collection('job_tasks'),
         attempts: @db.create_collection('job_task_attempts'),
 
-        job_events:     @db.create_collection('job_events',       capped_collection_opts),
-        task_events:    @db.create_collection('job_tasks_events', capped_collection_opts),
+        job_events:     @db.create_collection('job_events',      capped_collection_opts),
+        task_events:    @db.create_collection('job_task_events', capped_collection_opts),
       }
     end
 

data/scripts/hadoop_monitor/hadoopable.rb

@@ -1,12 +1,12 @@
 require 'stringio'
 
-module Vayacondios
+class Vayacondios
 
   module Hadoopable
 
     include Configurable
 
-    #--------------------------------------------------------------------------------    
+    #--------------------------------------------------------------------------------
     # Initialize jruby and tell it about hadoop.
     #--------------------------------------------------------------------------------
 
@@ -22,7 +22,7 @@ module Vayacondios
 
     $CLASSPATH << File.join(File.join(hadoop_home, 'conf') || ENV['HADOOP_CONF_DIR'],
                             '') # add trailing slash
-    
+
     Dir["#{hadoop_home}/{hadoop*.jar,lib/*.jar}"].each{|jar| require jar}
 
     include_class org.apache.hadoop.mapred.JobConf

data/scripts/hadoop_monitor/machine_monitor.rb

@@ -7,7 +7,7 @@ require 'scanf'
 require 'json'
 require 'mongo'
 
-module Vayacondios
+class Vayacondios
 
   class StatServer
 
@@ -60,7 +60,7 @@ module Vayacondios
 
       # main loop
       loop do
-        
+
         logger.debug "In main event loop. Waiting to see if the cluster is busy."
 
         # Get up-to-date on the state of the cluster.

data/spec/client/itemset_spec.rb

@@ -16,28 +16,28 @@ describe Vayacondios::Client::ItemSet do
     itemset = Vayacondios::Client::ItemSet.new("foohost", 9999, "fooorg", "footopic", "fooid")
     ary = ["foo", "bar", "baz"]
 
-    # Actually testing internals here to avoid 
+    # testing internals here to avoid shimming up HTTP libraries.
 
     it "generates a put request without a patch header when asked to create" do
-      req = itemset._req :create, ary
+      req = itemset.instance_eval{_req(:create, ary)}
 
       req.method.should eql('PUT')
-      req.body.should eql(ary.to_json)
+      req.body.should eql(MultiJson.encode(contents: ary))
       req.path.should eql('/v1/fooorg/itemset/footopic/fooid')
       req.each_header.to_a.should_not include(["x_method", "PATCH"])
     end
 
     it "generates a put request with a patch header when asked to update" do
-      req = itemset._req :update, ary
+      req = itemset.instance_eval{_req(:update, ary)}
 
       req.method.should eql('PUT')
-      req.body.should eql(ary.to_json)
+      req.body.should eql(MultiJson.encode(contents: ary))
       req.path.should eql('/v1/fooorg/itemset/footopic/fooid')
       req.each_header.to_a.should include(["x-method", "PATCH"])
     end
 
     it "generates a get request when asked to fetch" do
-      req = itemset._req :fetch
+      req = itemset.instance_eval{_req(:fetch)}
 
       req.method.should eql('GET')
       req.body.should be_nil
@@ -45,10 +45,10 @@ describe Vayacondios::Client::ItemSet do
     end
 
     it "generates a delete request when asked to remove" do
-      req = itemset._req :remove, ary
+      req = itemset.instance_eval{_req(:remove, ary)}
 
       req.method.should eql('DELETE')
-      req.body.should eql(ary.to_json)
+      req.body.should eql(MultiJson.encode(contents: ary))
       req.path.should eql('/v1/fooorg/itemset/footopic/fooid')
     end
   end

data/spec/server/itemset_spec.rb

@@ -157,7 +157,7 @@ describe HttpShim do
       with_api(HttpShim) do |api|
         get_request({:path => '/v1/infochimps/itemset/power/level'}, err) do |c|
           c.response_header.status.should == 200 
-          MultiJson.load(c.response).should eql(["foo", "bar"])
+          MultiJson.load(c.response).should eql({"contents" => ["foo", "bar"]})
         end
       end
     end
@@ -215,7 +215,7 @@ describe HttpShim do
       with_api(HttpShim) do |api|
         get_request({:path => '/v1/infochimps/itemset/merge/test'}, err) do |c|
           c.response_header.status.should == 200
-          MultiJson.load(c.response).should eql(["foo", "bar"])
+          MultiJson.load(c.response).should eql({"contents" => ["foo", "bar"]})
         end
       end
     end
@@ -277,7 +277,7 @@ describe HttpShim do
       with_api(HttpShim) do |api|
         get_request({:path => '/v1/infochimps/itemset/power/level'}, err) do |c|
           c.response_header.status.should == 200
-          MultiJson.load(c.response).should eql ["foo"]
+          MultiJson.load(c.response).should eql({"contents" => ["foo"]})
         end
       end
     end
@@ -304,7 +304,7 @@ describe HttpShim do
       with_api(HttpShim) do |api|
         get_request({:path => '/v1/infochimps/itemset/power/level'}, err) do |c|
           c.response_header.status.should == 200
-          MultiJson.load(c.response).should eql []
+          MultiJson.load(c.response).should eql({"contents" => []})
         end
       end
     end

data/src/main/java/com/infochimps/util/CurrentClass.java

@@ -0,0 +1,26 @@
+package com.infochimps.util;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CurrentClass extends SecurityManager {
+  private static CurrentClass SINGLETON = new CurrentClass();
+
+  // must call this directly. behavior is dependent on call stack
+  public static Class get() {
+    return SINGLETON.getCurrentClass();
+  }
+
+  // must call this directly. behavior is dependent on call stack
+  public static Logger getLogger() {
+    return LoggerFactory.getLogger(SINGLETON.getCurrentClass(2));
+  }
+
+  private Class getCurrentClass(int i) {
+    return getClassContext()[i];
+  }
+
+  private Class getCurrentClass() {
+    return getCurrentClass(3);
+  }
+}

data/src/main/java/com/infochimps/util/DebugUtil.java

@@ -0,0 +1,38 @@
+package com.infochimps.util;
+
+import java.net.InetSocketAddress;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+
+import java.security.cert.X509Certificate;
+import java.security.SecureRandom;
+import java.security.GeneralSecurityException;
+
+import java.net.Proxy;
+
+public class DebugUtil {
+    public static Proxy useCharles() {
+	trustAllCerts();
+	return new Proxy(Proxy.Type.HTTP, new InetSocketAddress("127.0.0.1", 8888));
+    }
+
+    public static void trustAllCerts() {
+	try {
+	    SSLContext sc = SSLContext.getInstance("SSL"); 
+	    sc.init(null,
+		    new TrustManager[] { 
+			new X509TrustManager() {     
+			    public X509Certificate[] getAcceptedIssuers() { return null;} 
+			    public void checkClientTrusted(X509Certificate[] certs, String authType) {} 
+			    public void checkServerTrusted(X509Certificate[] certs, String authType) {}
+			} 
+		    }, new SecureRandom()); 
+	    HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
+	} catch (GeneralSecurityException e) {}
+    }
+}

data/src/main/java/com/infochimps/util/HttpHelper.java

@@ -0,0 +1,112 @@
+package com.infochimps.util;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.zip.GZIPInputStream;
+
+import org.apache.commons.codec.binary.Base64;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static java.util.Map.Entry;
+
+public class HttpHelper {
+  private static final Base64 BASE64 = new Base64();
+  private static final boolean USE_CHARLES = false;
+
+  // opens or returns a null reader
+    public static BufferedReader openOrNull(Logger log, String urlString, Charset inputCharset) {
+    try { return open(log, urlString, inputCharset); }
+    catch (IOException e) {
+      log.warn("Got an exception trying to open {}: {}", urlString, e);
+      return null;
+    }
+  }
+
+  public static BufferedReader open(Logger log,
+				    String urlString,
+				    Charset inputCharset) throws IOException {
+    HttpURLConnection con = getConnection(urlString, log);
+    return getReader(con, log, inputCharset);
+  }
+
+  public static BufferedReader open(Logger log,
+				    String urlString,
+				    HashMap<String,String> extraHeaders,
+				    Charset inputCharset) throws IOException {
+
+    HttpURLConnection con = getConnection(urlString, log);
+    for (Entry<String,String> header : extraHeaders.entrySet())
+      con.setRequestProperty(header.getKey(), header.getValue());
+    return getReader(con, log, inputCharset);
+  }
+
+  private static HttpURLConnection getConnection(String urlString, Logger log) throws IOException {
+    URL url = null;
+    try { url = new URL(urlString); }
+    catch (MalformedURLException e) {
+      log.warn("malformed URL: {}", url);
+      throw new IOException(e);
+    }
+
+    HttpURLConnection con = (HttpURLConnection)(USE_CHARLES ?
+						url.openConnection(DebugUtil.useCharles()) :
+						url.openConnection());
+
+    String userInfo = url.getUserInfo();
+    if (userInfo != null) {
+      userInfo = URLDecoder.decode(userInfo, "US-ASCII");
+      con.setRequestProperty("Authorization", "Basic " + new String(BASE64.encodeBase64(userInfo.getBytes())));
+    }
+    con.setRequestProperty("Accept-Encoding", "gzip,deflate");
+    return con;
+  }
+
+  private static BufferedReader getReader(HttpURLConnection con,
+					  Logger log,
+					  Charset inputCharset) throws IOException {
+    InputStream in = null;
+
+    try { in = con.getInputStream(); }
+    catch (IOException e) {
+      // Some HTTP responses will raise an exception, but the
+      // useful information is in the error stream.
+
+      log.warn("Exception opening {}", con.getURL().toString());
+
+      InputStream errorStream = con.getErrorStream();
+      if (errorStream != null) {
+	BufferedReader r = new BufferedReader(new InputStreamReader(errorStream));
+	try { for (String line; (line = r.readLine()) != null; log.error(line)); }
+	catch (IOException nested_exc) {
+	  log.error("Got an exception in the exception handler: {}", nested_exc);
+	  throw e;
+	}
+      }
+      throw e;
+    }
+
+    String encoding = con.getContentEncoding();
+    log.debug("Got HTTP stream with content encoding type '" + encoding + "'");
+
+    if (encoding != null && encoding.equals("gzip")) in = new GZIPInputStream(in);
+
+    InputStreamReader istream_reader = new InputStreamReader(in, inputCharset);
+    BufferedReader reader = new BufferedReader(istream_reader);
+
+    log.debug("successfully opened connection to {} with character encoding {}",
+	      con.getURL().toString(),
+	      istream_reader.getEncoding());
+
+    return reader;
+  }
+}