async_enum 0.0.4

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MWU0MWNjY2UwYTE3OTI5Njc1OThiZTAyMDNjMjliNmQ3MWMzZmZjMg==
+  data.tar.gz: !binary |-
+    NGM5ZDA0NjA5NTI3M2YyNjVjZTA1N2UxOTVmYzI5MGQ2OGQ5YzhlNg==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    MjQ0Nzc5NmVlNTQzMzMzZmEyNzBmOTU4ZThkYWZmOWYwYmFmYTUxMDkxMTU5
+    NTFhNzI0MTUxNDQ2NGIyZjU3OTAyMTFiZTFiZjdmZjNlYWNhMGVlYjdkNTY2
+    OGFmZWM4OTk4MzUyNGUwYTQ3YWQ1N2Q3YjZjYWU2NjFkMmNlNTc=
+  data.tar.gz: !binary |-
+    MWQyYWM1ZWQ1ODc1YjQ0M2M2Y2VhMjQzM2I5ODhiODU0MjEzOTdmNzM0ZTY2
+    ZDcwMTE5YjhlYTEwZmUwMDBlMmE5MTI0MWY4MGU3ZWNlOTMxMzlhN2ZhOWIy
+    M2ExYjE1ZWI2MjM0MjBkMGM3ZTY1YWFiNDhjNmIwNmNmZWYzOTk=

data/.gitignore

@@ -0,0 +1,35 @@
+#********** osx template********** 
+
+.DS_Store
+
+# Thumbnails
+._*
+
+# Files that might appear on external disk
+.Spotlight-V100
+.Trashes
+ 
+
+#********** ruby template********** 
+
+*.gem
+Gemfile.lock
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+bin/
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/
+ 

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.0.0
+  - 1.9.3

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org/'
+
+gemspec

data/README.md

@@ -0,0 +1,121 @@
+# async\_enum
+
+Inspired by Enumerable#lazy coupled with Enumerator::Lazy, and the Async.js library, I thought it would be cool to have syntax like `urls.async.map{ |url| http_get(url) }`.
+
+![](https://fbcdn-sphotos-b-a.akamaihd.net/hphotos-ak-prn1/554187_10151609082562269_1115589261_n.jpg)
+
+#### Runs In Parallel
+
+```ruby
+require 'benchmark'
+
+sync = Benchmark.measure do
+	5.times.each{ sleep 0.1 }
+end
+
+async = Benchmark.measure do
+	5.times.async.each{ sleep 0.1 }
+end
+
+puts sync, async
+# 0.000000   0.000000   0.000000 (  0.500782)
+# 0.000000   0.010000   0.010000 (  0.102763)
+```
+
+#### How It Works
+
+The implementation was based on `Enumerable#lazy` introduced with Ruby 2.0. `Enumerator::Async` follows a similar approach, where the Enumerator is passed into the constructor. 
+
+```ruby
+enum = ('a'..'z').each
+
+# the following are equivalent
+
+Enumerator::Async.new(enum)
+enum.async
+```
+
+To get the enumerator back from the async enumerator, simply call `sync` or `to_enum` like so:
+
+```ruby
+enum.async.sync == enum
+# => true
+```
+
+Every operation on the async enumerator affects the contained enumerator in the same way. For instance:
+
+```ruby
+enum.with_index.to_a
+# => [ ['a', 0], ['b', 1], ['c', 2] ... ]
+
+enum.async.with_index.to_a
+# => [ ['a', 0], ['b', 1], ['c', 2] ... ]
+```
+
+Async methods can be chained just like tipical enumerator methods:
+
+```ruby
+enum.async.each{ sleep(0.1) }.each{ sleep(0.1) }
+```
+
+#### How to use it
+
+The method `Enumerable#async` was added so that every collection can be processed in parallel:
+
+```ruby
+[ 0, 1, 2 ].async.each{ |i| puts i }
+(0..2).async.map(&:to_i)
+
+# or chain to your heart's content
+
+(0..5).reject(&:even?).reverse.async.with_index.map{ |x, index| x + index }
+```
+
+#### Limiting thread pool size
+
+To limit the thread pool size, you can pass in an optional parameter to `async`. Suppose for performance reasons you want to use a maximum of 4 threads:
+
+```ruby
+(0..100).async(4).each do
+	# use bandwith
+end
+```
+
+#### Preventing race conditions
+
+When programming concurrently, nasty bugs can come up because some operations aren't atomic. For instance, incrementing a variable `x += 1` will not necessarily work as expected. To provide easy locking, there's a DSL-style `lock` method you can use in the block passed to the async enum. 
+
+```ruby
+count = 0
+('a'..'z').async.each do
+  lock :count do
+    count += 1
+  end
+end
+count
+# => 26
+```
+
+The name of the lock doesn't matter, but using the variable name helps make the code understandable. You should try to use 1 lock per thread-unsafe variable.
+
+## Notes
+
+To install it, add it to your gemfile:
+
+```
+# Gemfile
+
+gem 'async_enum', github: 'aj0strow/async_enum'
+```
+
+To run the demos, clone the project and run with the library included in the load path:
+
+```
+$ git clone git@github.com:aj0strow/async_enum
+$ cd async_enum
+$ ruby -I lib demos/sleep.rb 
+```
+
+**Disclaimer**: I am not an expert at multithreading. Quite the opposite in fact.
+
+Please report errors and feel free to contribute and improve things!

data/Rakefile

@@ -0,0 +1,9 @@
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/**/*.rb']
+  t.verbose = true
+end
+
+task default: :test

data/async_enum.gemspec

@@ -0,0 +1,21 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |s|
+  s.name        = 'async_enum'
+  s.version     = '0.0.4'
+  s.authors     = %w(aj0strow)
+  s.email       = 'alexander.ostrow@gmail.com'
+  s.description = 'iterate over enumerable objects concurrently'
+  s.summary     = 'Async Enumerable and Enumerator'
+  s.homepage    = 'http://github.com/aj0strow/async_enum'
+  s.license     = 'MIT'
+ 
+  s.files = `git ls-files`.split($/)
+  s.test_files  = s.files.grep(/test/)
+  s.require_paths = %w(lib)
+  
+  s.add_development_dependency 'bundler'
+  s.add_development_dependency 'rake'
+end

data/demos/lock.rb

@@ -0,0 +1,9 @@
+require 'async_enum'
+
+count = 0
+('a'..'z').async.each do
+  lock :count do
+    count += 1
+  end
+end
+puts count

data/demos/pool_size.rb

@@ -0,0 +1,12 @@
+require 'benchmark'
+require 'async_enum'
+
+default = Benchmark.measure do
+	500.times.async.each{ true }
+end
+
+limited = Benchmark.measure do
+  500.times.async(10).each{ true }
+end
+
+puts default, limited

data/demos/sleep.rb

@@ -0,0 +1,12 @@
+require 'benchmark'
+require 'async_enum'
+
+sync = Benchmark.measure do
+	5.times{ sleep 0.1 }
+end
+
+async = Benchmark.measure do
+  5.times.async.each{ sleep(0.1) } 
+end
+
+puts sync, async

data/demos/sync.rb

@@ -0,0 +1,4 @@
+require 'async_enum'
+
+enum = 5.times
+puts enum.async.sync == enum

data/lib/async_enum.rb

@@ -0,0 +1,8 @@
+require 'enumerator/async'
+
+module Enumerable
+  def async(pool_size = nil)
+    enum = self.is_a?(Enumerator) ? self : self.each
+    Enumerator::Async.new(enum, pool_size)
+  end
+end

data/lib/enumerator/async.rb

@@ -0,0 +1,106 @@
+class Enumerator
+  class Async < Enumerator
+    
+    class Lockset
+      def initialize
+        @semaphores = Hash.new do |locks, key| 
+          locks[key] = Mutex.new 
+        end
+      end
+      
+      def lock(key = :__default__, &thread_unsafe_block)
+        @semaphores[key].synchronize(&thread_unsafe_block)
+      end
+      
+      alias_method :evaluate, :instance_exec
+    end
+    
+    def initialize(enum, pool_size = nil)
+      @enum = enum
+      @pool_size = pool_size
+      @lockset = Lockset.new
+    end
+    
+    def to_a
+      @enum.to_a
+    end
+    
+    def sync
+      @enum
+    end
+    
+    alias_method :to_enum, :sync
+    
+    def size
+      @enum.size
+    end
+        
+    def each(&work)
+      raise_error('each') unless block_given?
+      
+      if @pool_size
+        threads = @pool_size.times.map do
+          Thread.new do
+            loop do
+              @enum.any? ? evaluate(*@enum.next, &work) : break
+            end
+          end
+        end
+        threads.each(&:join)
+        @enum.rewind
+      else
+        unlimited_threads(&work).each(&:join)
+      end
+      self
+    end
+    
+    def with_index(start = 0, &work)
+      @enum = @enum.with_index(start)
+      if block_given?
+        each(&work)
+      else
+        self
+      end
+    end
+    
+    def with_object(obj, &work)
+      @enum = @enum.with_object(obj)
+      if block_given?
+        each(&work); obj
+      else
+        self
+      end
+    end
+    
+    def map(&work)
+      raise_error('map') unless block_given?
+      
+      if @pool_size
+        outs = []
+        with_index do |item, index|
+          outs[index] = evaluate(item, &work)
+        end
+        outs
+      else
+        unlimited_threads(&work).map(&:value)
+      end
+    end
+
+    private
+    
+    def evaluate(*args, &work)
+      @lockset.instance_exec(*args, &work)
+    end
+    
+    def unlimited_threads(&work)
+      @enum.map do |*args|
+        Thread.new{ evaluate(*args, &work) }
+      end
+    end
+
+    def raise_error(method)
+      raise ArgumentError, "tried to call async #{method} without a block"
+    end
+    
+  end
+end

data/test/async_enum_test/benchmark_test.rb

@@ -0,0 +1,12 @@
+require 'test_helper'
+
+class BenchmarkTest < Test
+
+  test 'async is actually async' do
+    assert_performance_constant do |n|
+      [n, 500].min.times.async.each{ sleep(0.01) }
+    end
+  end
+  
+  
+end

data/test/async_enum_test/definitions_test.rb

@@ -0,0 +1,41 @@
+require 'test_helper'
+
+class DefinitionsTest < Test
+  
+  setup do
+    @methods = Enumerator::Async.instance_methods(false)
+  end
+  
+  test 'enumerator async exists' do
+    assert_equal 'constant', defined?(Enumerator::Async)
+  end
+  
+  test 'enumerator async inherits from enumerator' do
+    assert_includes Enumerator::Async.ancestors, Enumerator
+  end
+  
+  test 'enumerator holds onto enum' do
+    enum = Enumerator::Async.new(1..5)
+    assert_equal (1..5).to_a, enum.instance_variable_get('@enum').to_a
+  end
+  
+  test 'enumerator holds onto pool_size' do
+    enum = Enumerator::Async.new(1..5, 5)
+    assert_equal 5, enum.instance_variable_get('@pool_size')
+  end
+    
+  %w(to_a to_enum sync each map with_index with_object).each do |method|
+    test "enumerator async responds to #{method}" do
+      assert_includes @methods, method.to_sym
+    end
+  end
+  
+  test 'enumerable#async exists' do
+    assert_includes Enumerable.instance_methods(false), :async
+    assert 5.times.respond_to?(:async)
+    assert (1..5).respond_to?(:async)
+    assert [].respond_to?(:async)
+  end
+  
+  
+end

data/test/async_enum_test/enumerator_async_test.rb

@@ -0,0 +1,101 @@
+require 'test_helper'
+
+class EnumeratorAsyncTest < Test
+  
+  setup do
+    @enum = Enumerator::Async.new( (1..5).each )
+  end
+  
+  test 'to_a' do
+    assert_equal [1, 2, 3, 4, 5], @enum.to_a
+  end
+
+  test 'sync returns enumerator' do
+    refute @enum.sync.is_a?(Enumerator::Async)
+  end
+  
+  test 'to_enum alias of sync' do
+    assert_equal @enum.sync, @enum.to_enum
+  end
+  
+  test 'size' do
+    if Enumerator.instance_methods(false).include?(:size)
+      assert_equal 5, @enum.size
+    else
+      assert_raises(NoMethodError) { @enum.size }
+    end
+  end
+  
+  test 'each with no block' do
+    assert_raises(ArgumentError) { @enum.each }
+  end
+  
+  test 'each' do
+    nums = []
+    @enum.each do |i|
+      nums << i
+    end
+    assert_equal @enum.to_a, nums.sort
+  end
+  
+  test 'each with splatting' do
+    ranges = [ ('a'..'c').to_a ] * 3
+    enum = Enumerator::Async.new( ranges.each )
+    strs = []
+    enum.each do |a, b, c|
+      strs << (a + b + c)
+    end
+    assert_equal %w(abc abc abc), strs
+  end
+  
+  test 'with_index no block' do
+    pairs = @enum.with_index.to_a[0, 3]
+    assert_equal [[1, 0], [2, 1], [3, 2]], pairs
+  end
+  
+  test 'with_index' do
+    nums = []
+    @enum.with_index do |x, i|
+      nums << (x - i)
+    end 
+    assert_equal [1, 1, 1, 1, 1], nums
+  end
+  
+  test 'with_object no block' do
+    h = {}
+    pair = @enum.with_object(h).to_a.first
+    assert_equal [ 1, h ], pair
+  end
+  
+  test 'with_object' do
+    to_i = @enum.with_object({}) do |x, hash|
+      hash[ x.to_s ] = x
+    end
+    assert_equal to_i['3'], 3
+  end
+  
+  test 'map no block' do
+    assert_raises(ArgumentError) { @enum.map }
+  end
+  
+  test 'map' do
+    squares = @enum.map{ |i| i * i }
+    assert_equal [1, 4, 9, 16, 25], squares
+  end
+  
+  test 'map keeps order' do
+    strs = @enum.map{ |i| sleep rand; i.to_s }
+    assert_equal '1 2 3 4 5', strs.join(' ')
+  end
+  
+  test 'lock in block' do
+    count = 0
+    1000.times.async(5).each do
+      lock :count do
+        count += 1
+      end
+    end
+    assert_equal 1000, count
+  end
+
+end

data/test/async_enum_test/thread_pool_test.rb

@@ -0,0 +1,24 @@
+require 'test_helper'
+
+class ThreadPoolTest < Test
+  
+  test 'pools can speed things up' do
+    default_start = Time.now
+    500.times.async.each{ true }
+    default_delta = Time.now - default_start
+    
+    rated_start = Time.now
+    500.times.async(10).each{ true }
+    rated_delta = Time.now - rated_start
+    
+    assert rated_delta < default_delta
+  end
+  
+  test 'pools with ranges' do
+    vals = (0..7).async(4).map do |x|
+      x + 2
+    end
+    assert_equal [2, 3, 4, 5, 6, 7, 8, 9], vals
+  end
+  
+end

data/test/test_helper.rb

@@ -0,0 +1,35 @@
+require 'minitest/autorun'
+require 'minitest/benchmark'
+require 'async_enum'
+
+
+
+Test = MiniTest::Unit::TestCase
+
+
+
+def setup(&block)
+  define_method('setup', &block)
+end
+
+def test(test_name, &block)
+  define_method("test_#{ test_name.gsub(/\s+/, '_') }", &block)
+end
+
+def cleanup(&block)
+  define_method('cleanup', &block)
+end
+
+def teardown(&block)
+  define_method('teardown', &block)
+end
+
+
+
+def assert_singleton_method(object, sym)
+  assert object.respond_to?(sym), "Expected #{object} singleton to respond to #{sym}"
+end
+
+def assert_instance_method(object, sym)
+   assert object.method_defined?(sym), "Expected #{object} instance to respond to #{sym}"
+end

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: async_enum
+version: !ruby/object:Gem::Version
+  version: 0.0.4
+platform: ruby
+authors:
+- aj0strow
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-07-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: iterate over enumerable objects concurrently
+email: alexander.ostrow@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- README.md
+- Rakefile
+- async_enum.gemspec
+- demos/lock.rb
+- demos/pool_size.rb
+- demos/sleep.rb
+- demos/sync.rb
+- lib/async_enum.rb
+- lib/enumerator/async.rb
+- test/async_enum_test/benchmark_test.rb
+- test/async_enum_test/definitions_test.rb
+- test/async_enum_test/enumerator_async_test.rb
+- test/async_enum_test/thread_pool_test.rb
+- test/test_helper.rb
+homepage: http://github.com/aj0strow/async_enum
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.5
+signing_key: 
+specification_version: 4
+summary: Async Enumerable and Enumerator
+test_files:
+- test/async_enum_test/benchmark_test.rb
+- test/async_enum_test/definitions_test.rb
+- test/async_enum_test/enumerator_async_test.rb
+- test/async_enum_test/thread_pool_test.rb
+- test/test_helper.rb