pacer-parallel 0.1.0-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 45fecca778590a9eedc6c7386894dd14847fb538
+  data.tar.gz: 07d3840329d2aaa564b395313ce622395058c6ba
+SHA512:
+  metadata.gz: f1f15ee12ce896353781f578924cd6912f1bfda5d776c740a8ed79582e8dd9dcedc820b215263fd487bc35fe4cbdd507901a5d74559264c14f88b8594e94333f
+  data.tar.gz: 6dc2fab326c7ccfb7318e6be352b8309c0218f00fb13d0cfe5afc3203b9ba639a18aed6de70a8d93b877e8777d06537a268b456bfdbc0ca635288a61fed417fe

data/.gitignore

@@ -0,0 +1,18 @@
+target
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,5 @@
+source "http://rubygems.org"
+
+gemspec
+
+gem 'pacer', :path => '../pacer'

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Darrick Wiebe
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,70 @@
+# Pacer::Parallel
+
+Parallelize [Pacer](https://github.com/pangloss/pacer) Routes.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'pacer-parallel'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install pacer-parallel
+
+## Usage
+
+```ruby
+  g.v.parallel(threads: 8, in_buffer: 4, out_buffer: 10) do |v|
+    v.all(&:out)
+  end
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+
+## Notes
+
+
+* eagerly consume (1) input and push into a channel
+  * ChannelCapPipe
+  * create a cap pipe that does this. The pipe's output is the channel
+  * source data may be slow. Should probably not use a go block
+  * 1 thread in a loop
+* Control the construction of parallel pipes. Default 2 threads, make
+it configurable.
+  * standard copy split pipe can push the channel to subchannels
+  * each parallel route pulls from the channel.
+    * in a go block (waits will not block go thread pool)
+    * ChannelReaderPipe
+    * PathChannelReaderPipe
+  * parallel routes are unmodified
+  * cap each route - eagerly consume input and push into a channel
+    * ChannelCapPipe again
+  * ExhaustMergePipe + GatherPipe to create a route to an list of
+    channels
+  * use alts to read from any of the channels
+    * ChannelFanInPipe
+
+
+
+## Pipe structure built to create parallel route:
+
+ CCP
+ CSP (parallelism is 1 thread per pipe being split into)
+   CRP -> Work ... -> CCP
+   CRP -> Work ... -> CCP
+   ...
+ EMP
+ GP
+ CARP

data/Rakefile

@@ -0,0 +1,19 @@
+require "bundler/gem_tasks"
+
+task :package do
+  system 'lein', 'with-profile', 'release', 'uberjar'
+end
+
+file Pacer::Parallel::JAR_PATH => FileList['project.clj', 'src/clojure/**/*.clj', 'src/java/**/*.java'] do
+  Rake::Task['package'].execute
+end
+
+task :jar => Pacer::Parallel::JAR_PATH
+
+task :build => :jar
+task :install => :jar
+
+desc "Run the clojure test and integration suites"
+task :expectations do
+  sh "lein with-profile integration expectations"
+end

data/pacer-parallel.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../src/ruby', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'pacer/parallel/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "pacer-parallel"
+  spec.version       = Pacer::Parallel::VERSION
+  spec.platform    = 'java'
+  spec.authors       = ["Darrick Wiebe"]
+  spec.email         = ["dw@xnlogic.com"]
+  spec.description   = %q{Simple parallel routes in Pacer}
+  spec.summary       = %q{With the magic of Clojure's core.async}
+  spec.homepage      = "http://xnlogic.com"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/) + [Pacer::Parallel::JAR_PATH]
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["src/ruby"]
+
+  spec.add_dependency "pacer", "~> 1.3.3"
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

data/project.clj

@@ -0,0 +1,28 @@
+(defproject
+  pacer.parallel "0.1.0-SNAPSHOT"
+  :description "Leveraging core.async in Pacer"
+  :url         "http://xnlogic.com"
+  :license     {:name "MIT"}
+  :min-lein-version  "2.0.0"
+  :dependencies   [[org.clojure/clojure "1.5.1"]
+                   [core.async "0.1.0-SNAPSHOT"]
+                   [com.tinkerpop/pipes "2.3.0"]]
+  :global-vars {*warn-on-reflection* true}
+  :profiles
+  {:dev
+   {:source-paths      ["src/clojure"]
+    :java-source-paths ["src/java"]}
+   :release
+   {:source-paths      ["src/clojure"]
+    :java-source-paths ["src/java"]
+    :target-path       "target/release"}
+   :integration
+   {:source-paths      ["src/clojure"]
+    :java-source-paths ["src/java"]
+    :test-paths        ["test_integration"]
+    :target-path       "target/integration"}
+   }
+  :plugins [[lein-expectations "0.0.7"]
+            [lein-autoexpect "0.2.5"]
+            [lein-kibit "0.0.7"]
+            ])

data/src/clojure/pacer/parallel.clj

@@ -0,0 +1,23 @@
+(ns pacer.parallel
+  (:require [clojure.core.async :refer [chan >!! close! alts!!]])
+  (:import com.tinkerpop.pipes.Pipe))
+
+(defn pipe->chan [^Pipe pipe buffer]
+  (let [c (if buffer (chan buffer) (chan))]
+    (if (.hasNext pipe)
+      (future
+        (loop [v (.next pipe)]
+          (when v (>!! c v))
+          (if (.hasNext pipe)
+            (recur (.next pipe))
+            (close! c))))
+      (close! c))
+    c))
+
+(defn chan-select [chans]
+  (let [[v c] (alts!! chans)]
+    (if v
+      [v chans]
+      (if (= 1 (count chans))
+        nil
+        (recur (remove #{c} chans))))))

data/src/java/com/xnlogic/pacer/ChannelCapPipe.java

@@ -0,0 +1,47 @@
+package com.xnlogic.pacer;
+
+import com.tinkerpop.pipes.AbstractPipe;
+import com.tinkerpop.pipes.Pipe;
+import com.tinkerpop.pipes.transform.TransformPipe;
+import com.tinkerpop.pipes.util.FastNoSuchElementException;
+
+import clojure.lang.RT;
+import clojure.lang.Var;
+import clojure.lang.Symbol;
+
+public class ChannelCapPipe<S> extends AbstractPipe<S, Object> implements TransformPipe<S, Object> {
+  private static final Var REQUIRE = RT.var("clojure.core", "require"); 
+  private static final Var PIPE_TO_CHAN = RT.var("pacer.parallel", "pipe->chan");
+  private static boolean environmentReady = false;
+
+  public static void setupEnvironment() {
+    if (!ChannelCapPipe.environmentReady) {
+      REQUIRE.invoke(Symbol.intern(null, "clojure.core"));
+      REQUIRE.invoke(Symbol.intern(null, "pacer.parallel"));
+      REQUIRE.invoke(Symbol.intern(null, "clojure.core.async"));
+      ChannelCapPipe.environmentReady = true;
+    }
+  }
+
+
+  private Object buffer;
+  private boolean hasRun = false;
+
+  public ChannelCapPipe(Object buffer) {
+    ChannelCapPipe.setupEnvironment();
+    this.buffer = buffer;
+  }
+
+  protected Object processNextStart() {
+    if (!this.hasRun) {
+      this.hasRun = true;
+      return PIPE_TO_CHAN.invoke(starts, buffer);
+    } else {
+      throw FastNoSuchElementException.instance();
+    }
+  }
+
+  public void reset() {
+    this.hasRun = false;
+  }
+}

data/src/java/com/xnlogic/pacer/ChannelFanInPipe.java

@@ -0,0 +1,41 @@
+package com.xnlogic.pacer;
+
+import com.tinkerpop.pipes.AbstractPipe;
+import com.tinkerpop.pipes.Pipe;
+import com.tinkerpop.pipes.transform.TransformPipe;
+import com.tinkerpop.pipes.util.FastNoSuchElementException;
+
+import java.util.List;
+import clojure.lang.RT;
+import clojure.lang.Var;
+
+public class ChannelFanInPipe<S> extends AbstractPipe<List<Object>, S> implements TransformPipe<List<Object>, S> {
+  private static final Var VEC = RT.var("clojure.core", "vec");
+  private static final Var NTH = RT.var("clojure.core", "nth");
+  private static final Var CHAN_SELECT = RT.var("pacer.parallel", "chan-select");
+
+  private Object chans;
+
+  protected S processNextStart() {
+    while (true) {
+      if (this.chans == null) {
+        Object next = starts.next();
+        if (next != null)
+          this.chans = VEC.invoke(next);
+      } else {
+        Object vec = CHAN_SELECT.invoke(this.chans);
+        if (vec == null) {
+          this.chans = null;
+        } else {
+          this.chans = NTH.invoke(vec, 1);
+          return (S) NTH.invoke(vec, 0);
+        }
+      }
+    }
+  }
+
+  public void reset() {
+    this.chans = null;
+  }
+}
+

data/src/java/com/xnlogic/pacer/ChannelReaderPipe.java

@@ -0,0 +1,33 @@
+package com.xnlogic.pacer;
+
+import com.tinkerpop.pipes.AbstractPipe;
+import com.tinkerpop.pipes.Pipe;
+import com.tinkerpop.pipes.transform.TransformPipe;
+import com.tinkerpop.pipes.util.FastNoSuchElementException;
+
+import clojure.lang.RT;
+import clojure.lang.Var;
+
+public class ChannelReaderPipe<S> extends AbstractPipe<S, Object> implements TransformPipe<S, Object> {
+  private static final Var READ_CHAN = RT.var("clojure.core.async", "<!!");
+
+  private Object chan;
+
+  protected Object processNextStart() {
+    while (true) {
+      if (this.chan == null) {
+        this.chan = starts.next();
+      } else {
+        Object value = READ_CHAN.invoke(this.chan);
+        if (value == null)
+          this.chan = null;
+        else
+          return value;
+      }
+    }
+  }
+
+  public void reset() {
+    this.chan = null;
+  }
+}

data/src/ruby/pacer-parallel.rb

@@ -0,0 +1 @@
+require 'pacer/parallel'

data/src/ruby/pacer/clojure.rb

@@ -0,0 +1,13 @@
+module Pacer
+  module Clojure
+    import 'clojure.lang.RT'
+    import 'clojure.lang.Symbol'
+    REQUIRE = RT.var("clojure.core", "require");
+
+    class << self
+      def require(namespace)
+        REQUIRE.invoke(Symbol.intern(nil, namespace));
+      end
+    end
+  end
+end

data/src/ruby/pacer/parallel.rb

@@ -0,0 +1,28 @@
+require "pacer/parallel/version"
+require File.join(Pacer::Parallel::PATH, Pacer::Parallel::JAR_PATH)
+require "pacer/clojure"
+require 'pacer'
+require "pacer/parallel/channel_cap"
+require "pacer/parallel/channel_reader"
+require "pacer/parallel/channel_fan_in"
+
+module Pacer
+  module Routes
+    module RouteOperations
+      def parallel(opts = {}, &block)
+        threads = opts.fetch(:threads, 2)
+        if threads > 0
+          branched = (0...threads).reduce(channel_cap buffer: opts.fetch(:in_buffer, threads)) do |r, n|
+            r.branch do |x|
+              b = block.call x.channel_reader(based_on: self)
+              b.channel_cap buffer: opts[:out_buffer]
+            end
+          end
+          branched.merge_exhaustive.gather.channel_fan_in(based_on: block.call(self))
+        else
+          block.call self
+        end
+      end
+    end
+  end
+end

data/src/ruby/pacer/parallel/channel_cap.rb

@@ -0,0 +1,29 @@
+module Pacer
+  module Routes
+    module RouteOperations
+      # This creates a channel, emits it and spawns a thread
+      # that will push all data in the source into the channel.
+      def channel_cap(opts = {})
+        chain_route(opts.merge transform: Pacer::Parallel::ChannelCap, element_type: :channel)
+      end
+    end
+  end
+
+  module Parallel
+    module ChannelCap
+      import com.xnlogic.pacer.ChannelCapPipe
+
+      attr_accessor :buffer
+
+      protected
+
+      def attach_pipe(end_pipe)
+        pipe = ChannelCapPipe.new(buffer)
+        pipe.setStarts end_pipe if end_pipe
+        pipe
+      end
+    end
+  end
+end
+
+

data/src/ruby/pacer/parallel/channel_fan_in.rb

@@ -0,0 +1,27 @@
+module Pacer
+  module Routes
+    module RouteOperations
+      # The source of this should be a pipe emitting lists of channels.
+      def channel_fan_in(opts = {})
+        chain_route(opts.merge transform: Pacer::Parallel::ChannelFanIn)
+      end
+    end
+  end
+
+  module Parallel
+    module ChannelFanIn
+      import com.xnlogic.pacer.ChannelFanInPipe
+
+      protected
+
+      def attach_pipe(end_pipe)
+        pipe = ChannelFanInPipe.new
+        pipe.setStarts end_pipe if end_pipe
+        pipe
+      end
+    end
+  end
+end
+
+
+

data/src/ruby/pacer/parallel/channel_reader.rb

@@ -0,0 +1,24 @@
+module Pacer
+  module Routes
+    module RouteOperations
+      # The source of this in a pipe emitting individual channels
+      def channel_reader(opts = {})
+        chain_route(opts.merge transform: Pacer::Parallel::ChannelReader)
+      end
+    end
+  end
+
+  module Parallel
+    module ChannelReader
+      import com.xnlogic.pacer.ChannelReaderPipe
+
+      protected
+
+      def attach_pipe(end_pipe)
+        pipe = ChannelReaderPipe.new
+        pipe.setStarts end_pipe if end_pipe
+        pipe
+      end
+    end
+  end
+end

data/src/ruby/pacer/parallel/version.rb

@@ -0,0 +1,9 @@
+module Pacer
+  module Parallel
+    VERSION = "0.1.0"
+    JAR_VERSION = "#{VERSION}-SNAPSHOT"
+    JAR = "pacer.parallel-#{ JAR_VERSION }-standalone.jar"
+    JAR_PATH = "target/release/#{ JAR }"
+    PATH = File.expand_path(File.join(File.dirname(__FILE__), '../../../..'))
+  end
+end

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: pacer-parallel
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: java
+authors:
+- Darrick Wiebe
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2013-08-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: pacer
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.3
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: bundler
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rake
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+description: Simple parallel routes in Pacer
+email:
+- dw@xnlogic.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- pacer-parallel.gemspec
+- project.clj
+- src/clojure/pacer/parallel.clj
+- src/java/com/xnlogic/pacer/ChannelCapPipe.java
+- src/java/com/xnlogic/pacer/ChannelFanInPipe.java
+- src/java/com/xnlogic/pacer/ChannelReaderPipe.java
+- src/ruby/pacer-parallel.rb
+- src/ruby/pacer/clojure.rb
+- src/ruby/pacer/parallel.rb
+- src/ruby/pacer/parallel/channel_cap.rb
+- src/ruby/pacer/parallel/channel_fan_in.rb
+- src/ruby/pacer/parallel/channel_reader.rb
+- src/ruby/pacer/parallel/version.rb
+- target/release/pacer.parallel-0.1.0-SNAPSHOT-standalone.jar
+homepage: http://xnlogic.com
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- src/ruby
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.0.3
+signing_key:
+specification_version: 4
+summary: With the magic of Clojure's core.async
+test_files: []