vayacondios-server 0.1.12 → 0.2.1

data/src/main/java/com/infochimps/util/HttpHelper.java

@@ -33,28 +33,28 @@ public class HttpHelper {
   }
 
   public static BufferedReader open(Logger log,
-				    String urlString,
-				    Charset inputCharset) throws IOException {
+                                    String urlString,
+                                    Charset inputCharset) throws IOException {
     return getReader(openStream(log, urlString), inputCharset);
   }
 
   public static BufferedReader open(Logger log,
-				    String urlString,
-				    HashMap<String,String> extraHeaders,
-				    Charset inputCharset) throws IOException {
+                                    String urlString,
+                                    HashMap<String,String> extraHeaders,
+                                    Charset inputCharset) throws IOException {
     return getReader(openStream(log, urlString, extraHeaders), inputCharset);
 
   }
 
   public static InputStream openStream(Logger log,
-				       String urlString) throws IOException {
+                                       String urlString) throws IOException {
     HttpURLConnection con = getConnection(urlString, log);
     return getStream(con, log);
   }
 
   public static InputStream openStream(Logger log,
-				       String urlString,
-				       HashMap<String,String> extraHeaders) throws IOException {
+                                       String urlString,
+                                       HashMap<String,String> extraHeaders) throws IOException {
     HttpURLConnection con = getConnection(urlString, log);
     for (Entry<String,String> header : extraHeaders.entrySet())
       con.setRequestProperty(header.getKey(), header.getValue());
@@ -70,8 +70,8 @@ public class HttpHelper {
     }
 
     HttpURLConnection con = (HttpURLConnection)(USE_CHARLES ?
-						url.openConnection(DebugUtil.useCharles()) :
-						url.openConnection());
+                                                url.openConnection(DebugUtil.useCharles()) :
+                                                url.openConnection());
 
     String userInfo = url.getUserInfo();
     if (userInfo != null) {
@@ -83,7 +83,7 @@ public class HttpHelper {
   }
 
   private static InputStream getStream(HttpURLConnection con,
-				       Logger log) throws IOException {
+                                       Logger log) throws IOException {
     InputStream in = null;
 
     try { in = con.getInputStream(); }
@@ -95,12 +95,12 @@ public class HttpHelper {
 
       InputStream errorStream = con.getErrorStream();
       if (errorStream != null) {
-	BufferedReader r = new BufferedReader(new InputStreamReader(errorStream));
-	try { for (String line; (line = r.readLine()) != null; log.warn(line)); }
-	catch (IOException nested_exc) {
-	  log.error("Got an exception in the exception handler: {}", nested_exc);
-	  throw e;
-	}
+        BufferedReader r = new BufferedReader(new InputStreamReader(errorStream));
+        try { for (String line; (line = r.readLine()) != null; log.warn(line)); }
+        catch (IOException nested_exc) {
+          log.error("Got an exception in the exception handler: {}", nested_exc);
+          throw e;
+        }
       }
       throw e;
     }

data/src/main/java/com/infochimps/vayacondios/ItemSets.java

@@ -23,6 +23,7 @@ import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.io.IOException;
 import java.lang.reflect.Type;
+import java.lang.reflect.InvocationTargetException;
 import java.net.HttpURLConnection;
 import java.util.ArrayList;
 import java.util.List;
@@ -35,12 +36,17 @@ import org.slf4j.Logger;
  * This is the first level of the Vayacondios path-building hierarchy
  * that is capable of directly manipulating itemsets.
  */
-public class ItemSets extends Organization {
+public class ItemSets<LinkType extends LinkToVCD> extends Organization {
   public ItemSets(PathBuilder delegate) { super(delegate); }
 
-  public ItemSets(Organization org) {
+  /**
+   * @param linkClass for testing purposes. can be used to shim up a
+   * dummy vayacondios session.
+   */
+  public ItemSets(Organization org, LinkType linkToVCD) {
     super(org);
     _org = org;
+    (_vcdLink = linkToVCD).setParent(this);
   }
 
   //----------------------------------------------------------------------------
@@ -69,44 +75,7 @@ public class ItemSets extends Organization {
    * @return a collection of items
    */
   public List<Item> fetch(String topic, String id) throws IOException {
-    BufferedReader reader = null;
-    try {
-      reader = openUrl(urlString(PATH_COMPONENT, topic, id));
-    } catch (FileNotFoundException ex) {
-      // In the case of a 404, return an empty set.
-      return new ArrayList();
-    }
-    String line = reader.readLine();
-    JsonElement response;
-    JsonElement itemSet;
-
-    ArrayList<Item> result = new ArrayList<Item>();
-
-    // assume Vayacondios response comes in a single line
-    if (line != null &&
-	(response = PARSER.parse(line)).isJsonObject() &&
-	(itemSet = (response.getAsJsonObject().get("contents"))).isJsonArray()) {
-      for (JsonElement elem : itemSet.getAsJsonArray()) {
-	if (!elem.isJsonPrimitive()) {
-	  LOG.warn("ignoring non-primitive in itemset: " + elem);
-	  continue;
-	}
-
-	JsonPrimitive item = elem.getAsJsonPrimitive();
-	if      (item.isBoolean()) result.add(new Item(item.getAsBoolean()));
-	else if (item.isNumber())  result.add(new Item(item.getAsNumber()));
-	else if (item.isString())  result.add(new Item(item.getAsString()));
-
-	else LOG.warn("ignoring unrecognized type in itemset: " + item);
-      }
-    }
-    
-    if ((line = reader.readLine()) != null)
-      LOG.warn("expected eof but saw " + line);
-
-    reader.close();
-
-    return result;
+    return _vcdLink.fetch(topic, id);
   }
 
   /**
@@ -121,8 +90,8 @@ public class ItemSets extends Organization {
    * @param items items whose existence should be ensured in the set.
    */
   public void create(String topic,
-		     String id,
-		     List<Item> items) throws IOException {
+                     String id,
+                     List<Item> items) throws IOException {
     mutate("PUT", topic, id, items);
   }
 
@@ -137,8 +106,8 @@ public class ItemSets extends Organization {
    * @param items items whose absence should be ensured in the set.
    */
   public void remove(String topic,
-		     String id,
-		     List<Item> items) throws IOException {
+                     String id,
+                     List<Item> items) throws IOException {
     mutate("DELETE", topic, id, items);
   }
 
@@ -154,8 +123,8 @@ public class ItemSets extends Organization {
    * @param items items whose existence should be ensured in the set.
    */
   public void update(String topic,
-		     String id,
-		     List<Item> items) throws IOException {
+                     String id,
+                     List<Item> items) throws IOException {
     mutate("PATCH", topic, id, items);
   }
 
@@ -170,75 +139,16 @@ public class ItemSets extends Organization {
   //----------------------------------------------------------------------------
 
   protected void mutate(String method,
-			String topic,
-			String id,
-			List<Item> items) throws IOException {
-
-    // serialize the items
-    HashMap content = new HashMap();
-    content.put("contents", items);
-    String body = GSON.toJson(content);
-    // connect to our standard path
-    URL url = new URL(urlString(PATH_COMPONENT, topic, id));
-    HttpURLConnection connection = (HttpURLConnection)
-      ((Boolean.valueOf(System.getProperty("ics.http.use_charles"))) ? 
-       url.openConnection(DebugUtil.useCharles()) : url.openConnection());
-
-    // configure connection
-    connection.setDoOutput(true);
-
-    // NOTE: Uncommenting this (and not calling
-    // connection.getInputStream()) causes Java to hang without
-    // sending its payload.
-
-    // connection.setDoInput(false);
-
-    if (method.equals("DELETE")) {
-      connection.setRequestMethod("PUT");
-      connection.setRequestProperty("X-Method", "DELETE");
-    } else if (method.equals("PATCH")) {
-      connection.setRequestMethod("PUT");
-      connection.setRequestProperty("X-Method", "PATCH");
-    } else connection.setRequestMethod(method);
-    connection.setRequestProperty("Content-Type", "application/json"); 
-    connection.setRequestProperty("Accept", "*/*");
-    connection.setRequestProperty("Content-Length",
-				  Integer.toString(body.getBytes().length));
-    connection.setUseCaches(false);
-
-    LOG.debug("sending: " + body);
-    LOG.debug("via " +
-	      connection.getRequestMethod() +
-	      " to " +
-	      urlString(PATH_COMPONENT, topic, id));
-
-    // connect and write
-    OutputStream os = connection.getOutputStream();
-    os.write(body.getBytes("UTF-8"));
-    os.flush();
-    os.close();
-
-    // ignore reponse
-    InputStream is = connection.getInputStream();
-
-    LOG.trace("ignoring response from Vayacondios.");
-    byte buf[] = new byte[256];
-    while (is.read(buf) != -1);
-    LOG.trace("response ignored.");
-    is.close();
-
-    // fin.
-    connection.disconnect();
+                        String topic,
+                        String id,
+                        List<Item> items) throws IOException {
+    _vcdLink.mutate(method, topic, id, items);
   }
 
+  private LinkType _vcdLink;
   private Organization _org;
 
-  private static final JsonParser PARSER	= new JsonParser();
-  private static final Logger LOG		= getLogger();
-  private static final String PATH_COMPONENT    = "itemset";
-  private static final Gson GSON                = new GsonBuilder().
-    registerTypeAdapter(Item.class, new Item.Serializer()).
-    create();
+  private static final Logger LOG               = getLogger();
 
   //============================================================================
   // Topic
@@ -393,10 +303,10 @@ public class ItemSets extends Organization {
   public static class Item {
     static class Serializer implements JsonSerializer {
       public JsonElement serialize(Object item,
-				   Type typeOfId,
-				   JsonSerializationContext context) {
-	return GSON.toJsonTree(Item.class.isAssignableFrom(item.getClass()) ? 
-			       ((Item)item).getObject() : item);
+                                   Type typeOfId,
+                                   JsonSerializationContext context) {
+        return GSON.toJsonTree(Item.class.isAssignableFrom(item.getClass()) ? 
+                               ((Item)item).getObject() : item);
       }
       private static final Gson GSON = new Gson();
       private static final Logger LOG = getLogger();
@@ -419,19 +329,19 @@ public class ItemSets extends Organization {
 
     public Boolean getAsBoolean() {
       if (_type != TYPE.BOOLEAN)
-	throw new ClassCastException("item is not a boolean");
+        throw new ClassCastException("item is not a boolean");
       return (Boolean)_item;
     }
 
     public Number getAsNumber() {
       if (_type != TYPE.NUMBER)
-	throw new ClassCastException("item is not a number");
+        throw new ClassCastException("item is not a number");
       return (Number)_item;
     }
 
     public String getAsString() {
       if (_type != TYPE.STRING)
-	throw new ClassCastException("item is not a string");
+        throw new ClassCastException("item is not a string");
       return (String)_item;
     }
 
@@ -449,7 +359,7 @@ public class ItemSets extends Organization {
 
     public boolean equals(Object other) {
       return (Item.class.isAssignableFrom(other.getClass())) ? 
-	_item.equals(((Item)other).getObject()) : _item.equals(other);
+        _item.equals(((Item)other).getObject()) : _item.equals(other);
     }
 
     //--------------------------------------------------------------------------

data/src/main/java/com/infochimps/vayacondios/LinkToVCD.java

@@ -0,0 +1,18 @@
+package com.infochimps.vayacondios;
+
+import java.util.List;
+import java.io.IOException;
+import static com.infochimps.vayacondios.ItemSets.Item;
+
+public abstract class LinkToVCD {
+  public abstract List<Item> fetch(String topic, String id) throws IOException;
+  public abstract void mutate(String method,
+                              String topic,
+                              String id,
+                              List<Item> items) throws IOException;
+
+  protected ItemSets getParent() { return _parent; }
+  public void setParent(ItemSets parent) { _parent = parent; }
+
+  private ItemSets _parent;
+}

data/src/main/java/com/infochimps/vayacondios/MemoryVCDShim.java

@@ -0,0 +1,84 @@
+package com.infochimps.vayacondios;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonIOException;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonSerializer;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSyntaxException;
+
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import java.net.URL;
+
+import org.slf4j.Logger;
+
+import static com.infochimps.util.CurrentClass.getLogger;
+import static com.infochimps.vayacondios.ItemSets.Item;
+
+public class MemoryVCDShim extends LinkToVCD {
+  public MemoryVCDShim() {
+    _topics = new HashMap<String, Map<String, List<Item>>>();
+  }
+
+  public List<Item> fetch(String topicName, String id) {
+    Map<String, List<Item>> topic = _topics.get(topicName);
+    List<Item> result;
+
+    LOG.trace("topics before fetch: {}", _topics);
+
+    if (topic == null || (result = topic.get(id)) == null) {
+      LOG.debug("couldn't find {}.{}. returning empty list", topicName, id);
+      return new ArrayList<Item>();
+    } else {
+      LOG.debug("returning {}.{} => {}", topicName, id, result);
+      return result;
+    }
+  }
+
+  public void mutate(String method,
+                     String topicName,
+                     String id,
+                     List<Item> items) {
+    LOG.trace("topics before mutate: {}", _topics);
+
+    Map<String, List<Item>> topic = _topics.get(topicName);
+    if (topic == null) {
+      LOG.debug("creating topic {}.", topicName);
+      _topics.put(topicName, topic = new HashMap<String, List<Item>>());
+    }
+
+    if (method.equals("DELETE")) {
+      LOG.trace("removing {} from {}.{}", items, topicName, id);
+      topic.get(id).removeAll(items);
+    }
+    else if (method.equals("PATCH")) {
+      LOG.trace("adding {} to {}.{}", items, topicName, id);
+      topic.get(id).addAll(items);
+    }
+    else if (method.equals("PUT")) {
+      LOG.trace("creating {}.{} with {}", topicName, id, items);
+      topic.put(id, items);
+    }
+
+    LOG.trace("topics after mutate: {}", _topics);
+  }
+
+  private Map<String, Map<String, List<Item>>> _topics;
+  private static final Logger LOG = getLogger();
+}

data/src/main/java/com/infochimps/vayacondios/Organization.java

@@ -6,7 +6,8 @@ import java.util.Collection;
  * An organization is the last commmon class in the Vayacondios
  * hierarchy between the itemset and stash interface.
  */
-public class Organization extends VayacondiosClient {
+public class Organization<LinkType extends LinkToVCD>
+  extends VayacondiosClient {
   public Organization(PathBuilder delegate) { super(delegate); }
 
   public Organization(VayacondiosClient server, String organization) {
@@ -22,7 +23,19 @@ public class Organization extends VayacondiosClient {
   /**
    * @return new ItemSets path builder for this organization
    */
-  public ItemSets itemsets() { return new ItemSets(this); }
+  public ItemSets<LinkType> itemsets() {
+    return itemsets(new StandardVCDLink());
+  }
+
+
+  /**
+   * @param linkType link type to use to 
+   * 
+   * @return new ItemSets path builder for this organization
+   */
+  public ItemSets<LinkType> itemsets(LinkToVCD linkToVCD) {
+    return new ItemSets(this, linkToVCD);
+  }
 
   //----------------------------------------------------------------------------
   // API HTTP path components

data/src/main/java/com/infochimps/vayacondios/StandardVCDLink.java

@@ -0,0 +1,152 @@
+package com.infochimps.vayacondios;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonIOException;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonSerializer;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSyntaxException;
+
+
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import java.net.URL;
+
+import org.slf4j.Logger;
+
+import com.infochimps.util.DebugUtil;
+import com.infochimps.util.HttpHelper;
+import static com.infochimps.util.CurrentClass.getLogger;
+import static com.infochimps.vayacondios.ItemSets.Item;
+
+public class StandardVCDLink extends LinkToVCD {
+  public List<Item> fetch(String topic, String id) throws IOException {
+    BufferedReader reader = null;
+    try {
+      reader = openUrl(getParent().urlString(PATH_COMPONENT, topic, id));
+    } catch (FileNotFoundException ex) {
+      // In the case of a 404, return an empty set.
+      return new ArrayList();
+    }
+    String line = reader.readLine();
+    JsonElement response;
+    JsonElement itemSet;
+
+    ArrayList<Item> result = new ArrayList<Item>();
+
+    // assume Vayacondios response comes in a single line
+    if (line != null &&
+        (response = PARSER.parse(line)).isJsonObject() &&
+        (itemSet = (response.getAsJsonObject().get("contents"))).isJsonArray()) {
+      for (JsonElement elem : itemSet.getAsJsonArray()) {
+        if (!elem.isJsonPrimitive()) {
+          LOG.warn("ignoring non-primitive in itemset: " + elem);
+          continue;
+        }
+
+        JsonPrimitive item = elem.getAsJsonPrimitive();
+        if      (item.isBoolean()) result.add(new Item(item.getAsBoolean()));
+        else if (item.isNumber())  result.add(new Item(item.getAsNumber()));
+        else if (item.isString())  result.add(new Item(item.getAsString()));
+
+        else LOG.warn("ignoring unrecognized type in itemset: " + item);
+      }
+    }
+    
+    if ((line = reader.readLine()) != null)
+      LOG.warn("expected eof but saw " + line);
+
+    reader.close();
+
+    return result;
+  }
+
+  public void mutate(String method,
+                     String topic,
+                     String id,
+                     List<Item> items) throws IOException {
+    
+    // serialize the items
+    HashMap content = new HashMap();
+    content.put("contents", items);
+    String body = GSON.toJson(content);
+    // connect to our standard path
+    URL url = new URL(getParent().urlString(PATH_COMPONENT, topic, id));
+    HttpURLConnection connection = (HttpURLConnection)
+      ((Boolean.valueOf(System.getProperty("ics.http.use_charles"))) ? 
+       url.openConnection(DebugUtil.useCharles()) : url.openConnection());
+
+    // configure connection
+    connection.setDoOutput(true);
+
+    // NOTE: Uncommenting this (and not calling
+    // connection.getInputStream()) causes Java to hang without
+    // sending its payload.
+
+    // connection.setDoInput(false);
+
+    if (method.equals("DELETE")) {
+      connection.setRequestMethod("PUT");
+      connection.setRequestProperty("X-Method", "DELETE");
+    } else if (method.equals("PATCH")) {
+      connection.setRequestMethod("PUT");
+      connection.setRequestProperty("X-Method", "PATCH");
+    } else connection.setRequestMethod(method);
+    connection.setRequestProperty("Content-Type", "application/json"); 
+    connection.setRequestProperty("Accept", "*/*");
+    connection.setRequestProperty("Content-Length",
+                                  Integer.toString(body.getBytes().length));
+    connection.setUseCaches(false);
+
+    LOG.debug("sending: " + body);
+    LOG.debug("via " +
+              connection.getRequestMethod() +
+              " to " +
+              getParent().urlString(PATH_COMPONENT, topic, id));
+
+    // connect and write
+    OutputStream os = connection.getOutputStream();
+    os.write(body.getBytes("UTF-8"));
+    os.flush();
+    os.close();
+
+    // ignore reponse
+    InputStream is = connection.getInputStream();
+
+    LOG.trace("ignoring response from Vayacondios.");
+    byte buf[] = new byte[256];
+    while (is.read(buf) != -1);
+    LOG.trace("response ignored.");
+    is.close();
+
+    // fin.
+    connection.disconnect();
+  }
+
+  private BufferedReader openUrl(String urlString) throws IOException {
+    HashMap headers = new HashMap();
+    headers.put("Accept", "*/*");
+    return HttpHelper.open(LOG, urlString, headers, Charset.forName("UTF-8"));
+  }
+  
+  private static final Gson GSON                = new GsonBuilder().
+    registerTypeAdapter(Item.class, new Item.Serializer()).
+    create();
+  private static final JsonParser PARSER        = new JsonParser();
+  private static final Logger LOG               = getLogger();
+  private static final String PATH_COMPONENT    = "itemset";
+}

data/src/main/java/com/infochimps/vayacondios/VCDIntegrationTest.java

@@ -76,7 +76,7 @@ public class VCDIntegrationTest {
 
   public static void main(String argv[]) {
     System.out.println("*** If Vayacondios is not running on port " + VCD_PORT + ", " +
-		       "this will fail. ***");
+                       "this will fail. ***");
     System.out.println("Running Vayacondios integration test...");
 
     try {

data/src/main/java/com/infochimps/vayacondios/VayacondiosClient.java

@@ -45,9 +45,9 @@ public class VayacondiosClient extends PathBuilder {
   //----------------------------------------------------------------------------
 
   protected String urlString(String organization,
-			     String type,
-			     String topic,
-			     String id) {
+                             String type,
+                             String topic,
+                             String id) {
     return new StringBuilder().
       append("http://").
       append(getServerName()).
@@ -67,16 +67,6 @@ public class VayacondiosClient extends PathBuilder {
   protected int    getPort()       { return _port;       }
   protected String getServerName() { return _serverName; }
 
-  //----------------------------------------------------------------------------
-  // private methods
-  //----------------------------------------------------------------------------
-
-  protected BufferedReader openUrl(String urlString) throws IOException {
-    HashMap headers = new HashMap();
-    headers.put("Accept", "*/*");
-    return HttpHelper.open(LOG, urlString, headers, Charset.forName("UTF-8"));
-  }
-  
   //----------------------------------------------------------------------------
   // fields
   //----------------------------------------------------------------------------