pacer-parallel 0.1.0-java → 0.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 45fecca778590a9eedc6c7386894dd14847fb538
-  data.tar.gz: 07d3840329d2aaa564b395313ce622395058c6ba
+  metadata.gz: 49501e461707e82d8442a765a9d85e5c3d6116fa
+  data.tar.gz: 14c307841f53a72eac941344f8aed4da07e333db
 SHA512:
-  metadata.gz: f1f15ee12ce896353781f578924cd6912f1bfda5d776c740a8ed79582e8dd9dcedc820b215263fd487bc35fe4cbdd507901a5d74559264c14f88b8594e94333f
-  data.tar.gz: 6dc2fab326c7ccfb7318e6be352b8309c0218f00fb13d0cfe5afc3203b9ba639a18aed6de70a8d93b877e8777d06537a268b456bfdbc0ca635288a61fed417fe
+  metadata.gz: 1e6f8de6c827d83f70752055cccdd6e46c2af7831320725fe2b01164bc4e941fef876ea501e8a356e6fe0b5ad211f1fa609c9c416143bc6f95be298a205df5c2
+  data.tar.gz: 1ec244e49d3b9a50518ae2df7634da3715c11ccf715be4802e9f81eafd68ca8483d4252500569796034c46be0c511b6c58539ccc56fff71cf83b30b0dd703cda

data/README.md

@@ -2,6 +2,23 @@
 
 Parallelize [Pacer](https://github.com/pangloss/pacer) Routes.
 
+## Usage
+
+The following will create a route that will do the operation in the block in 2 parallel threads, while consuming the source data and
+enqueueing it for consumption by those threads in one additional thread:
+
+```ruby
+  g.v.parallel { |v| v.some_expensive_operation }
+```
+
+There are a few simple options, all optional. By default in_buffer is equal to the number of threads, and output is not buffered.
+
+```ruby
+  g.v.parallel(threads: 8, in_buffer: 4, out_buffer: 10) do |v|
+    v.all(&:out)
+  end
+```
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -16,20 +33,10 @@ Or install it yourself as:
 
     $ gem install pacer-parallel
 
-## Usage
-
-```ruby
-  g.v.parallel(threads: 8, in_buffer: 4, out_buffer: 10) do |v|
-    v.all(&:out)
-  end
-```
-
 ## Contributing
 
 1. Fork it
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
+...
 5. Create new Pull Request
 
 
@@ -45,21 +52,22 @@ Or install it yourself as:
 it configurable.
   * standard copy split pipe can push the channel to subchannels
   * each parallel route pulls from the channel.
-    * in a go block (waits will not block go thread pool)
-    * ChannelReaderPipe
-    * PathChannelReaderPipe
+      * in a go block (waits will not block go thread pool)
+      * ChannelReaderPipe
+      * PathChannelReaderPipe
   * parallel routes are unmodified
   * cap each route - eagerly consume input and push into a channel
-    * ChannelCapPipe again
+      * ChannelCapPipe again
   * ExhaustMergePipe + GatherPipe to create a route to an list of
     channels
   * use alts to read from any of the channels
-    * ChannelFanInPipe
+      * ChannelFanInPipe
 
 
 
 ## Pipe structure built to create parallel route:
 
+```
  CCP
  CSP (parallelism is 1 thread per pipe being split into)
    CRP -> Work ... -> CCP
@@ -67,4 +75,5 @@ it configurable.
    ...
  EMP
  GP
- CARP
+ CFIP
+```

data/project.clj

@@ -1,5 +1,5 @@
 (defproject
-  pacer.parallel "0.1.0-SNAPSHOT"
+  pacer.parallel "0.2.0-SNAPSHOT"
   :description "Leveraging core.async in Pacer"
   :url         "http://xnlogic.com"
   :license     {:name "MIT"}

data/src/clojure/pacer/parallel.clj

@@ -14,6 +14,18 @@
       (close! c))
     c))
 
+(defn pipe->path-chan [^Pipe pipe buffer]
+  (let [c (if buffer (chan buffer) (chan))]
+    (if (.hasNext pipe)
+      (future
+        (loop [v (.next pipe) path (.getCurrentPath pipe)]
+          (when v (>!! c path))
+          (if (.hasNext pipe)
+            (recur (.next pipe) (.getCurrentPath pipe))
+            (close! c))))
+      (close! c))
+    c))
+
 (defn chan-select [chans]
   (let [[v c] (alts!! chans)]
     (if v

data/src/java/com/xnlogic/pacer/ChannelCapPipe.java

@@ -8,10 +8,13 @@ import com.tinkerpop.pipes.util.FastNoSuchElementException;
 import clojure.lang.RT;
 import clojure.lang.Var;
 import clojure.lang.Symbol;
+import java.util.List;
+import java.util.ArrayList;
 
 public class ChannelCapPipe<S> extends AbstractPipe<S, Object> implements TransformPipe<S, Object> {
   private static final Var REQUIRE = RT.var("clojure.core", "require"); 
   private static final Var PIPE_TO_CHAN = RT.var("pacer.parallel", "pipe->chan");
+  private static final Var PIPE_TO_PATH_CHAN = RT.var("pacer.parallel", "pipe->path-chan");
   private static boolean environmentReady = false;
 
   public static void setupEnvironment() {
@@ -23,7 +26,6 @@ public class ChannelCapPipe<S> extends AbstractPipe<S, Object> implements Transf
     }
   }
 
-
   private Object buffer;
   private boolean hasRun = false;
 
@@ -35,12 +37,23 @@ public class ChannelCapPipe<S> extends AbstractPipe<S, Object> implements Transf
   protected Object processNextStart() {
     if (!this.hasRun) {
       this.hasRun = true;
-      return PIPE_TO_CHAN.invoke(starts, buffer);
+      if (this.pathEnabled) {
+        return PIPE_TO_PATH_CHAN.invoke(starts, buffer);
+      } else {
+        return PIPE_TO_CHAN.invoke(starts, buffer);
+      }
     } else {
       throw FastNoSuchElementException.instance();
     }
   }
 
+  // If path is enabled, the path will be pushed into the channel.
+  // Having a path available here as well causes the merge pipe to
+  // produce a messed up path.
+  public List getCurrentPath() {
+    return new ArrayList();
+  }
+
   public void reset() {
     this.hasRun = false;
   }

data/src/java/com/xnlogic/pacer/ChannelFanInPipe.java

@@ -15,6 +15,7 @@ public class ChannelFanInPipe<S> extends AbstractPipe<List<Object>, S> implement
   private static final Var CHAN_SELECT = RT.var("pacer.parallel", "chan-select");
 
   private Object chans;
+  private List currentPath;
 
   protected S processNextStart() {
     while (true) {
@@ -28,13 +29,28 @@ public class ChannelFanInPipe<S> extends AbstractPipe<List<Object>, S> implement
           this.chans = null;
         } else {
           this.chans = NTH.invoke(vec, 1);
-          return (S) NTH.invoke(vec, 0);
+          if (this.pathEnabled) {
+            this.currentPath = (List) NTH.invoke(vec, 0);
+            System.out.println(this.currentPath);
+            return (S) this.currentPath.get(this.currentPath.size() - 1);
+          } else {
+            return (S) NTH.invoke(vec, 0);
+          }
         }
       }
     }
   }
 
+  public List getCurrentPath() {
+    if (this.pathEnabled) {
+      return this.currentPath;
+    } else {
+      throw new RuntimeException(Pipe.NO_PATH_MESSAGE);
+    }
+  }
+
   public void reset() {
+    this.currentPath = null;
     this.chans = null;
   }
 }

data/src/java/com/xnlogic/pacer/ChannelReaderPipe.java

@@ -7,27 +7,52 @@ import com.tinkerpop.pipes.util.FastNoSuchElementException;
 
 import clojure.lang.RT;
 import clojure.lang.Var;
+import java.util.List;
 
-public class ChannelReaderPipe<S> extends AbstractPipe<S, Object> implements TransformPipe<S, Object> {
+public class ChannelReaderPipe<S> extends AbstractPipe<Object, S> implements TransformPipe<Object, S> {
   private static final Var READ_CHAN = RT.var("clojure.core.async", "<!!");
 
   private Object chan;
+  private List currentPath;
 
-  protected Object processNextStart() {
+  protected S processNextStart() {
     while (true) {
       if (this.chan == null) {
         this.chan = starts.next();
       } else {
-        Object value = READ_CHAN.invoke(this.chan);
+        Object value;
+        if (this.pathEnabled) {
+          this.currentPath = (List) READ_CHAN.invoke(this.chan);
+          if (this.currentPath != null) {
+            value = this.currentPath.get(this.currentPath.size() - 1);
+          } else {
+            value = null;
+          }
+        } else {
+          value = READ_CHAN.invoke(this.chan);
+        }
         if (value == null)
           this.chan = null;
         else
-          return value;
+          return (S) value;
       }
     }
   }
 
+  public List getCurrentPath() {
+    if (this.pathEnabled) {
+      return this.currentPath;
+    } else {
+      throw new RuntimeException(Pipe.NO_PATH_MESSAGE);
+    }
+  }
+
+  protected List getPathToHere() {
+    return this.currentPath;
+  }
+
   public void reset() {
+    this.currentPath = null;
     this.chan = null;
   }
 }

data/src/ruby/pacer/parallel/version.rb

@@ -1,6 +1,7 @@
 module Pacer
   module Parallel
-    VERSION = "0.1.0"
+    # Be sure to update project.clj
+    VERSION = "0.2.0"
     JAR_VERSION = "#{VERSION}-SNAPSHOT"
     JAR = "pacer.parallel-#{ JAR_VERSION }-standalone.jar"
     JAR_PATH = "target/release/#{ JAR }"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pacer-parallel
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: java
 authors:
 - Darrick Wiebe
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-08-27 00:00:00.000000000 Z
+date: 2013-08-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pacer
@@ -77,7 +77,7 @@ files:
 - src/ruby/pacer/parallel/channel_fan_in.rb
 - src/ruby/pacer/parallel/channel_reader.rb
 - src/ruby/pacer/parallel/version.rb
-- target/release/pacer.parallel-0.1.0-SNAPSHOT-standalone.jar
+- target/release/pacer.parallel-0.2.0-SNAPSHOT-standalone.jar
 homepage: http://xnlogic.com
 licenses:
 - MIT