threadify_procs 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 68ea99f978e760abe63d79a225b5a62ba609d58a
+  data.tar.gz: 65b85ee7bbe065f5ef827c76fd44c93886237a6e
+SHA512:
+  metadata.gz: f4780d7d718692e686c93e508dec9d0bdad3c7b775f5d912b82b4a38e068b07187f9d819d05f29dbf88b2d0d554968de0df4775db6ed5889140893efd8f64eb5
+  data.tar.gz: 84b1e44be3f747f6ddb22676d77456481878d93f68b33a1f8d3e8f907a713e61c7ae1c4a36881275c01765bb0db9a699e31f76d1679476953ecd10197e3e900f

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - 2.0.0
+  - 2.1.2
+script: bundle exec rspec

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in threadify_procs.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Antoine Qu'hen
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,28 @@
+# ThreadifyProcs
+
+Create an array of Procs, launch them within threads. It adresses the common
+problem of writting files concurrently in threads with a ruby proces or
+downloading simultaneously multiple files. It avoids 'Too many open files'
+errors.
+
+## Usage
+
+    require 'threadify_procs'
+    procs = [
+      Proc.new { puts 1 },
+      Proc.new { puts 2 }
+    ]
+    call_with_threads procs, number_of_threads: 50
+
+An other option is available :with_writer (boolean). Its goal is to launch
+another thread which responsibility is to create files on disc.
+
+    require 'threadify_procs'
+    procs = []
+    10_000.times do |n|
+      Proc.new do
+        @files_to_write << [
+          "#{Rails.root}/tmp/#{n}.txt", SecureRandom.uuid]
+      end
+    end
+    launch_in_threads procs, number_of_threads: 50, with_writer: true

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/threadify_procs/version.rb

@@ -0,0 +1,3 @@
+module ThreadifyProcs
+  VERSION = "0.0.1"
+end

data/lib/threadify_procs.rb

@@ -0,0 +1,89 @@
+require "threadify_procs/version"
+
+module ThreadifyProcs
+
+  def call_with_threads procs, options={}
+    set_procs procs
+    set_options options
+    with_writer_thread do
+      launch_procs
+    end
+  end
+
+  private
+
+  def launch_writer_thread
+    @writer_thread = Thread.new do
+      begin
+        while true do
+          unless @files_to_write.empty?
+            data = @files_to_write.shift
+            filename, string = data
+            File.open(filename, 'w') do |file|
+              file.binmode
+              file.write string
+            end
+          else
+            break if ready_to_kill_writer_thread?
+          end
+        end
+      rescue => exception
+        Thread.main.raise exception
+      end
+    end
+  end
+
+  def kill_writer_thread
+    @kill_writer_thread = true
+    @writer_thread.join
+  end
+
+  def launch_procs
+    @threads = [].tap do |threads|
+      groups_of_procs.each do |procs|
+        threads << Thread.new do
+          procs.each(&:call)
+        end
+      end
+    end
+    @threads.each(&:join)
+  end
+
+  def set_procs procs
+    @procs = procs
+    unless procs.kind_of?(Array) && procs.all?{|_proc|_proc.kind_of?(Proc)}
+      raise 'ThreadifyProcs expects an array of procs.'
+    end
+  end
+
+  def set_options options
+    @number_of_threads = options[:number_of_threads].to_i || 25
+    if @number_of_threads < 1
+      raise 'ThreadifyProcs expects a positive number of Threads.'
+    end
+    @procs_per_thread = (procs.size / @number_of_threads).ceil
+    @with_writer = !!options[:with_writer]
+    @files_to_write = [] if @with_writer
+  end
+
+  def with_writer_thread
+    begin
+      launch_writer_thread if @with_writer
+      yield
+    ensure
+      kill_writer_thread if @with_writer
+    end
+  end
+
+  def groups_of_procs
+    [].tap do |groups|
+      while !@procs.empty? do
+        groups << @procs.shift(@procs_per_thread)
+      end
+    end
+  end
+
+  def ready_to_kill_writer_thread?
+    @kill_writer_thread && @files_to_write.empty? && @threads.all?(&:stop?)
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+require 'bundler/setup'
+Bundler.setup
+
+require 'threadify_procs'
+
+RSpec.configure do |config|
+  # some (optional) config here
+end

data/spec/threadify_procs_spec.rb

@@ -0,0 +1,72 @@
+require 'spec_helper'
+require 'fileutils'
+
+class ThreadifiedJob
+  include ThreadifyProcs
+  attr_reader :total
+
+  def initialize
+    @total = 0
+  end
+
+  def procs
+    [].tap do |_procs|
+      3.times do |n|
+        _procs << Proc.new do
+          @total += n+1
+        end
+      end
+    end
+  end
+
+  def launch
+    call_with_threads procs, number_of_threads: 2
+  end
+end
+
+class ThreadifiedJobWithWriter < ThreadifiedJob
+
+  def procs
+    [].tap do |_procs|
+      3.times do |n|
+        _procs << Proc.new do
+          @files_to_write << ["#{File.dirname(__FILE__)}/../tmp/#{n}.txt", n]
+        end
+      end
+    end
+  end
+
+  def launch
+    @files_to_write = []
+    call_with_threads procs, number_of_threads: 3, with_writer: true
+  end
+end
+
+describe ThreadifyProcs do
+  let(:tmp_dir) { "#{File.dirname(__FILE__)}/../tmp" }
+
+  describe 'call_with_threads' do
+    subject{ ThreadifiedJob.new }
+
+    it 'should create threads from procs' do
+      subject.launch
+      expect(subject.total).to eq 6
+    end
+
+    describe 'with_writer' do
+      subject{ ThreadifiedJobWithWriter.new }
+      before { FileUtils.mkdir tmp_dir}
+      after { FileUtils.rm_r tmp_dir}
+
+      it 'should create threads from procs' do
+        subject.launch
+
+        3.times do |n|
+          path = "#{tmp_dir}/#{n}.txt"
+          expect File.exists?(path)
+          expect(File.read(path)).to eq n.to_s
+        end
+      end
+    end
+  end
+end

data/threadify_procs.gemspec

@@ -0,0 +1,22 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'threadify_procs/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'threadify_procs'
+  spec.version       = ThreadifyProcs::VERSION
+  spec.authors       = ["Antoine Qu'hen"]
+  spec.email         = ["antoinequhen@gmail.com"]
+  spec.summary       = %q{Launch an array of Procs within threads.}
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: threadify_procs
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Antoine Qu'hen
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- antoinequhen@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/threadify_procs.rb
+- lib/threadify_procs/version.rb
+- spec/spec_helper.rb
+- spec/threadify_procs_spec.rb
+- threadify_procs.gemspec
+homepage: 
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.1
+signing_key: 
+specification_version: 4
+summary: Launch an array of Procs within threads.
+test_files:
+- spec/spec_helper.rb
+- spec/threadify_procs_spec.rb
+has_rdoc: