datacraft 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0d5b93cedf58b0affe9e22abc1f1597a3de90c4e
-  data.tar.gz: d7592ee90f256ad37cd2c6b8b90a5b71cc397581
+  metadata.gz: 5f512f220800ff71938c3d2dfbe7c640dab90f13
+  data.tar.gz: aad2c68ec6f1663b074dff6c4981d4594588c560
 SHA512:
-  metadata.gz: e0eb59fa1bb9d2cb546425393bd07f97c754c343a4d685eeb2e11b3c55061617ff1ec22626a16814bd5a0c0ce29e188205834af64f8a570dd85abf566d2a7116
-  data.tar.gz: 2cb72b1a6852382c61985f620c8542744af97dab13df5091d443e810cac397af54a46525e86a44169381b00748533a2eaa3c2b168a00f1fe90dd09a60d6596b0
+  metadata.gz: f1f18984edf137b33982301aadb15c7a78a268b1cc50bde6ed3ad1939315b6eb9dc456cd23b268be13a443c5eb1e508c3fe3341b9d5ad0bb71ab7e0643a607bf
+  data.tar.gz: 9d172bf503e1576535550d60a764114fc33f900f63cf234ef73ce7419d60e37879e4f4a0fa42f9aac601b39e915fb505b09349e29e70e6b8ca639fc54d05d574

data/datacraft.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'thor', '~> 0'
+  spec.add_dependency 'thread', '~> 0'
 
   spec.add_development_dependency 'bundler', '~> 1.10'
   spec.add_development_dependency 'rake', '~> 10.0'

data/lib/datacraft/context.rb

@@ -1,5 +1,11 @@
 module Datacraft
   class Context
+    def initialize
+      options[:parallel] = false
+      options[:benchmark] = false
+      options[:n_threads] = 8
+    end
+
     def providers
       @providers ||= ProviderRegistry.new
     end

data/lib/datacraft/registry.rb

@@ -11,7 +11,7 @@ module Datacraft
   # common registry
   class Registry
     extend Forwardable
-    def_delegators :instances, :each, :map
+    def_delegators :instances, :each, :map, :size
     def initialize
       @items = []
     end

data/lib/datacraft/runner.rb

@@ -5,33 +5,23 @@ module Datacraft
   module Runner
     # run the instruction
     def run(instruction)
-      context = instruction.context
+      @context = instruction.context
       measurements = []
       measurements << Benchmark.measure('pre build:') do
-        context.pre_hooks.each(&:call)
+        @context.pre_hooks.each(&:call)
       end
       measurements << Benchmark.measure('process rows:') do
-        if context.options[:parallel]
-          pprocess_rows(
-            context.providers,
-            context.tweakers,
-            context.consumers)
-        else
-          process_rows(
-            context.providers,
-            context.tweakers,
-            context.consumers)
-        end
+        @context.options[:parallel] ? pprocess_rows : process_rows
       end
 
       measurements << Benchmark.measure('build:') do
-        build context.consumers
+        build @context.consumers
       end
 
       measurements << Benchmark.measure('post build:') do
-        context.post_hooks.each(&:call)
+        @context.post_hooks.each(&:call)
       end
-      report measurements if context.options[:benchmark]
+      report measurements if @context.options[:benchmark]
     end
 
     # output benchmark results
@@ -44,35 +34,37 @@ module Datacraft
     end
 
     # process rows sequentially
-    def process_rows(providers, tweakers, consumers)
-      providers.each do |provider|
+    def process_rows
+      @context.providers.each do |provider|
         provider.each do |row|
-          tweakers.each do |tweaker|
-            row = tweaker.tweak row
-            break unless row
-          end
-          # nil means to dismiss the row
-          next unless row
-          consumers.each do |consumer|
-            consumer << row
-          end
+          process row
         end
       end
     end
 
+    # tweak & consume one row
+    def process(row)
+      @context.tweakers.each do |tweaker|
+        row = tweaker.tweak row
+        return nil unless row
+      end
+      @context.consumers.each do |consumer|
+        consumer << row
+      end
+    end
+
     # process rows in parallel
-    def pprocess_rows(providers, tweakers, consumers)
-      threads = providers.map do |provider|
-        Thread.new(provider) do |p|
-          p.each do |row|
-            tweakers.each do |tweaker|
-              row = tweaker.tweak row
-              break unless row
-            end
-            next unless row
-            consumers.each do |consumer|
-              consumer << row
-            end
+    def pprocess_rows
+      thread_number = [@context.providers.size,
+                       @context.options[:n_threads]].min
+      queue = Queue.new
+      @context.providers.each { |p| queue << p }
+      threads = thread_number.times.map do |tn|
+        Thread.new do
+          until queue.empty?
+            puts ">> thread #{tn} working..."
+            p = queue.pop(true)
+            p.each { |row| process row }
           end
         end
       end

data/lib/datacraft/version.rb

@@ -1,3 +1,3 @@
 module Datacraft
-  VERSION = "0.2.0"
+  VERSION = '0.3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datacraft
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Xiaoxing Hu
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-07-27 00:00:00.000000000 Z
+date: 2015-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: thread
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement