datacraft 0.3.2 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d74a3c5f9f45ac21db0385ffedc9f3a7bf4b6d30
-  data.tar.gz: 11519d1e16e3adc961f6d961e3c417a22fee051d
+  metadata.gz: 2d9abea0dcb9fc8da790558bc9d848c5420e42e0
+  data.tar.gz: 114b10e2d87db5adada974f86b506a1ccd007e52
 SHA512:
-  metadata.gz: e359f0949e8ba8bf623c3e005401c8327e69a54d1c87e84722ec42a75a078d69e8968451071d85969578202b85dcf2681979bb6d5a38abce346c7a78b2f0e8e8
-  data.tar.gz: 5138f7d55179f9a35e54a4fd41f34850d4f78e48db119228eed9a848ceff8ee769359cf7822c980c2723ef8c52b6a35369cb0303f3fe812705cbbe48065d9319
+  metadata.gz: 1c812f178e2f5da84826fa70584ac66ca5fdccb0f92de6e505ba3c9253d74ecaa6ac93c25fb34eba8cf770d7418207c235de4a18d56d4b1192bf308c11c54f3e
+  data.tar.gz: 274b7e3e6022f75021e52272d9c08d2e980077444a5e1fadc9c42881628c7ba539510dece0e892880a973a8e05c68163424c00ea68c904e864958f9605a19296

data/lib/datacraft.rb

@@ -1,6 +1,6 @@
 require 'datacraft/version'
 require 'datacraft/exceptions'
-require 'datacraft/registry'
+require 'datacraft/definition'
 require 'datacraft/context'
 require 'datacraft/instruction'
 require 'datacraft/runner'

data/lib/datacraft/cli.rb

@@ -9,9 +9,15 @@ module Datacraft
       Datacraft.run instruction
     end
 
-    desc 'check [INSTRUCTION_FILE]', 'evaluate the instruction without running it'
+    desc 'check [INSTRUCTION_FILE]',
+         'evaluate the instruction without running it'
     def check(filename)
-      puts Instruction.check filename
+      begin
+        Instruction.from_file filename
+        puts 'You are ready to go.'
+      rescue InvalidInstruction => e
+        puts e
+      end
     end
   end
 end

data/lib/datacraft/context.rb

@@ -1,33 +1,38 @@
 module Datacraft
-  class Context
-    def initialize
-      options[:parallel] = false
-      options[:benchmark] = false
-      options[:n_threads] = 8
-    end
-
-    def providers
-      @providers ||= ProviderRegistry.new
-    end
-
-    def consumers
-      @consumers ||= ConsumerRegistry.new
+  # define the context of the instruction
+  module Context
+    def sources
+      @sources ||= []
     end
 
     def tweakers
-      @tweakers ||= TweakerRegistry.new
+      @tweakers ||= []
     end
 
     def pre_hooks
-      @pre_hooks ||= HookRegistry.new
+      @pre_hooks ||= []
     end
 
     def post_hooks
-      @post_hooks ||= HookRegistry.new
+      @post_hooks ||= []
+    end
+
+    def consumers
+      @consumers ||= []
     end
 
     def options
       @options ||= {}
     end
+
+    def validate
+      fail InvalidInstruction, "Please define data source with keyword 'from'." unless sources.size > 0
+      fail InvalidInstruction, "Please define data consumer with keyword 'to'." unless consumers.size > 0
+      sources.each(&:validate)
+      consumers.each(&:validate)
+      tweakers.each(&:validate)
+      pre_hooks.each(&:validate)
+      post_hooks.each(&:validate)
+    end
   end
 end

data/lib/datacraft/definition.rb

@@ -0,0 +1,108 @@
+require 'forwardable'
+
+module Datacraft
+
+  # for normalizing blocks
+  class CompatiableProc < Proc
+    # alias_method :run, :call
+    # alias_method :process, :call
+    alias_method :tweak, :call
+  end
+
+  # method management
+  module MethodDef
+    extend Forwardable
+    attr_reader :mandatory, :optional
+
+    def def_mandatory(*methods)
+      @mandatory = methods
+      methods.each do |method|
+        def_delegator :instance, method
+      end
+    end
+
+    def def_optional(*methods)
+      @optional = methods
+      methods.each do |method|
+        def_delegator :instance, method
+      end
+    end
+
+    def validate_methods(klass)
+      mandatory.each do |m|
+        fail InvalidInstruction, "Class <#{klass}> missing mandatory methods: #{m}." unless klass.method_defined?(m)
+      end
+    end
+  end
+
+  # definition of component
+  module Definition
+    extend Forwardable
+    def_delegator :instance, :respond_to?
+    def self.included(base)
+      base.extend(Forwardable)
+      base.extend(MethodDef)
+    end
+
+    def initialize(d)
+      @d = d
+    end
+
+    def instance
+      @instance ||= create_instance
+    end
+
+    private
+
+    def create_instance
+      if @d[:klass]
+        @d[:klass].new(*@d[:args])
+      elsif @d[:block]
+        CompatiableProc.new(&@d[:block])
+      end
+    end
+  end
+
+  # data source that provide data
+  class Source
+    include Definition
+    def_mandatory :each
+
+    def validate
+      fail InvalidSource, 'Source needs to be a class.' unless @d[:klass]
+      self.class.validate_methods @d[:klass]
+    end
+  end
+
+  # data consumer that consume data
+  class Consumer
+    include Definition
+    def_mandatory :<<
+    def_optional :build, :close
+
+    def validate
+      fail InvalidConsumer, 'Consumer needs to be a class.' unless @d[:klass]
+      self.class.validate_methods @d[:klass]
+    end
+  end
+
+  # tweaking data row
+  class Tweaker
+    include Definition
+    def_mandatory :tweak
+
+    def validate
+      self.class.validate_methods @d[:klass] if @d[:klass]
+    end
+  end
+
+  # pre/post build hooks
+  class Hook
+    include Definition
+    def_mandatory :call
+
+    def validate
+      fail InvalidHook, 'Hook has to be a block.' unless @d[:block]
+    end
+  end
+end

data/lib/datacraft/exceptions.rb

@@ -1,3 +1,7 @@
 module Datacraft
   class InvalidInstruction < StandardError; end
+  class InvalidSource < InvalidInstruction; end
+  class InvalidConsumer < InvalidInstruction; end
+  class InvalidTweaker < InvalidInstruction; end
+  class InvalidHook < InvalidInstruction; end
 end

data/lib/datacraft/instruction.rb

@@ -1,52 +1,45 @@
 module Datacraft
   # Building Instruction
   class Instruction
+    include Context
     attr_reader :context
 
-    def initialize(context = Context.new)
-      @context = context
+    def initialize
+      options[:parallel] = false
+      options[:benchmark] = false
+      options[:n_threads] = 8
     end
 
     def pre_build(&block)
-      @context.pre_hooks << { block: block }
+      pre_hooks << Hook.new(block: block)
     end
 
     def from(klass, *initialization_params)
-      @context.providers << { klass: klass, args: initialization_params }
+      sources << Source.new(klass: klass, args: initialization_params)
     end
 
     def tweak(klass = nil, *initialization_params, &block)
-      @context.tweakers << { klass: klass, args: initialization_params, block: block }
+      tweakers << Tweaker.new(klass: klass, args: initialization_params, block: block)
     end
 
     def to(klass, *initialization_params)
-      @context.consumers << { klass: klass, args: initialization_params }
+      consumers << Consumer.new(klass: klass, args: initialization_params)
     end
 
     def post_build(&block)
-      @context.post_hooks << { block: block }
+      post_hooks << Hook.new(block: block)
     end
 
     def set(key, value)
-      @context.options[key.to_sym] = value
+      options[key.to_sym] = value
     end
 
     def self.from_file(filename)
       script_content = IO.read(filename)
       instruction = Instruction.new
       instruction.instance_eval(script_content)
+      instruction.validate
       instruction
     end
-
-    def self.check(filename)
-      begin
-        script_content = IO.read(filename)
-        instruction = Instruction.new
-        instruction.instance_eval(script_content)
-        'Looks good.'
-      rescue InvalidInstruction => e
-        e.message
-      end
-    end
   end
 end

data/lib/datacraft/runner.rb

@@ -5,23 +5,33 @@ module Datacraft
   module Runner
     # run the instruction
     def run(instruction)
-      @context = instruction.context
+      @inst = instruction
       measurements = []
-      measurements << Benchmark.measure('pre build:') do
-        @context.pre_hooks.each(&:call)
+
+      # run pre_build hooks
+      if @inst.respond_to? :pre_hooks
+        measurements << Benchmark.measure('pre build:') do
+          @inst.pre_hooks.each(&:call)
+        end
       end
+
+      # process the rows
       measurements << Benchmark.measure('process rows:') do
-        @context.options[:parallel] ? pprocess_rows : process_rows
+        @inst.options[:parallel] ? pprocess_rows : process_rows
       end
 
+      # build
       measurements << Benchmark.measure('build:') do
-        build @context.consumers
+        build @inst.consumers
       end
 
-      measurements << Benchmark.measure('post build:') do
-        @context.post_hooks.each(&:call)
+      # run post_build hooks
+      if @inst.respond_to? :post_hooks
+        measurements << Benchmark.measure('post build:') do
+          @inst.post_hooks.each(&:call)
+        end
       end
-      report measurements if @context.options[:benchmark]
+      report measurements if @inst.options[:benchmark]
     end
 
     # output benchmark results
@@ -35,8 +45,8 @@ module Datacraft
 
     # process rows sequentially
     def process_rows
-      @context.providers.each do |provider|
-        provider.each do |row|
+      @inst.sources.each do |source|
+        source.each do |row|
           process row
         end
       end
@@ -44,21 +54,21 @@ module Datacraft
 
     # tweak & consume one row
     def process(row)
-      @context.tweakers.each do |tweaker|
+      @inst.tweakers.each do |tweaker|
         row = tweaker.tweak row
         return nil unless row
       end
-      @context.consumers.each do |consumer|
+      @inst.consumers.each do |consumer|
         consumer << row
       end
     end
 
     # process rows in parallel
     def pprocess_rows
-      thread_number = [@context.providers.size,
-                       @context.options[:n_threads]].min
+      thread_number = [@inst.sources.size,
+                       @inst.options[:n_threads]].min
       queue = Queue.new
-      @context.providers.each { |p| queue << p }
+      @inst.sources.each { |p| queue << p }
       threads = thread_number.times.map do
         Thread.new do
           begin
@@ -67,10 +77,6 @@ module Datacraft
             end
           rescue ThreadError
           end
-          # until queue.empty?
-          #   p = queue.pop(true)
-          #   p.each { |row| process row }
-          # end
         end
       end
       threads.each(&:join)

data/lib/datacraft/version.rb

@@ -1,3 +1,3 @@
 module Datacraft
-  VERSION = '0.3.2'
+  VERSION = '0.4.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datacraft
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
 platform: ruby
 authors:
 - Xiaoxing Hu
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-07-30 00:00:00.000000000 Z
+date: 2015-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -119,9 +119,9 @@ files:
 - lib/datacraft.rb
 - lib/datacraft/cli.rb
 - lib/datacraft/context.rb
+- lib/datacraft/definition.rb
 - lib/datacraft/exceptions.rb
 - lib/datacraft/instruction.rb
-- lib/datacraft/registry.rb
 - lib/datacraft/runner.rb
 - lib/datacraft/version.rb
 homepage: https://github.com/xiaoxinghu/datacraft

data/lib/datacraft/registry.rb

@@ -1,74 +0,0 @@
-require 'forwardable'
-
-module Datacraft
-  # for normalizing blocks
-  class CompatiableProc < Proc
-    alias_method :run, :call
-    alias_method :process, :call
-    alias_method :tweak, :call
-  end
-
-  # common registry
-  class Registry
-    extend Forwardable
-    def_delegators :instances, :each, :map, :size
-    def initialize
-      @items = []
-    end
-
-    attr_accessor :mandatory_methods
-
-    def instances
-      @instances ||= @items.map do |i|
-        if i[:klass]
-          i[:klass].new(*i[:args])
-        elsif i[:block]
-          CompatiableProc.new(&i[:block])
-        end
-      end
-    end
-
-    def <<(val)
-      klass = val[:klass]
-      block = val[:block]
-      if klass
-        fail InvalidInstruction, "#{klass.name} needs to implement methods: "\
-          "#{mandatory_methods}" unless valid? klass
-      elsif block
-      else
-        fail 'registry error'
-      end
-      @items << val
-    end
-
-    def valid?(klass)
-      mandatory_methods.all? do |m|
-        klass.method_defined? m
-      end
-    end
-  end
-
-  class ProviderRegistry < Registry
-    def mandatory_methods
-      [:each]
-    end
-  end
-
-  class ConsumerRegistry < Registry
-    def mandatory_methods
-      [:<<]
-    end
-  end
-
-  class TweakerRegistry < Registry
-    def mandatory_methods
-      [:tweak]
-    end
-  end
-
-  class HookRegistry < Registry
-    def mandatory_methods
-      [:run]
-    end
-  end
-end