datacraft 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2d9abea0dcb9fc8da790558bc9d848c5420e42e0
-  data.tar.gz: 114b10e2d87db5adada974f86b506a1ccd007e52
+  metadata.gz: bc6e2cf4c695b0b7f6ff9544415a3bcf929ac3f6
+  data.tar.gz: 1c1ab63f47629926fd21a2c4a919defc6d030519
 SHA512:
-  metadata.gz: 1c812f178e2f5da84826fa70584ac66ca5fdccb0f92de6e505ba3c9253d74ecaa6ac93c25fb34eba8cf770d7418207c235de4a18d56d4b1192bf308c11c54f3e
-  data.tar.gz: 274b7e3e6022f75021e52272d9c08d2e980077444a5e1fadc9c42881628c7ba539510dece0e892880a973a8e05c68163424c00ea68c904e864958f9605a19296
+  metadata.gz: 814dfe30c18efa74e89ef332e14159ef5077e344baee32bc870c18eb6b045456f35c51c7c3cbdcf1e1c18e1bd85c5f2babf96ac47192a70c04bd28ca02e1df7c
+  data.tar.gz: 12ac00e4e1ba23433cf4b78d570cf15921c3dc459fba9628b2913401020f58d5cf040230207551db61e82513304bbda0908a67ef3a58181930206af1cac0be39

data/README.md

@@ -7,6 +7,21 @@ Play with data like a Pro, and have fun like Minecraft.
 [![Dependency Status](https://gemnasium.com/xiaoxinghu/datacraft.svg)](https://gemnasium.com/xiaoxinghu/datacraft)
 [![Code Climate](https://codeclimate.com/github/xiaoxinghu/datacraft/badges/gpa.svg)](https://codeclimate.com/github/xiaoxinghu/datacraft)
 
+## what is it
+
+`Datacraft` is an ETL tool highly inspired by open source project [kiba](https://github.com/thbar/kiba). But we need certain advanced features which current kiba does not provide, so we decided to roll out our own version. Here are the major ones are already implemented:
+
+- build action after loading data (explained later)
+- parallel processing
+- lazy initialization
+
+And we need more. Here are some ideas bouncing around in my head and might become new features.
+
+- data flow, pipelining
+- dry run with analytic results
+- snapshot the data on change points and generate report for intuitive debugging.
+
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -25,7 +40,109 @@ Or install it yourself as:
 
 ## Usage
 
-    $ dcraft build inst.rb
+Here is an example of instruction script.
+
+```ruby
+
+# data source class
+class CsvFile
+  def initialize(csv_file)
+    @csv = CSV.open(csv_file, headers: true, header_converters: :symbol)
+  end
+
+  # mandatory method for data source class.
+  # Should always yield data rows.
+  def each
+    @csv.each do |row|
+      yield(row.to_hash)
+    end
+    @csv.close
+  end
+end
+
+# Data consumer class. Here we just output statistic
+# information of the data set.
+class ReportBuilder
+  def initialize
+    @total_employee = 0
+    @total_age = 0
+  end
+
+  # mandatory method for consume data
+  def <<(row)
+    @total_employee += 1
+    @total_age += row[:age].to_i
+  end
+
+  # optional method for build final product
+  def build
+    File.open('report.txt', 'w') do |f|
+      f.puts "Total Employee: #{@total_employee}"
+      f.puts "Total Age: #{@total_age}"
+      f.puts "Average Age: #{@total_age / @total_employee}" unless @total_employee == 0
+    end
+  end
+end
+
+retirement_age = 60
+# define data source
+from CsvFile, 'employees.csv'
+
+total_rows = 0
+# tweak each row
+tweak do |row|
+  # eliminate the retired ones, return nil means to
+  # discard the data
+  total_rows += 1
+  row[:age].to_i < retirement_age ? row : nil
+end
+
+# define data consumer
+to ReportBuilder
+```
+
+Run the script:
+
+```bash
+$ dcraft build inst.rb
+```
+
+### parallel processing
+
+To improve the performance of the script, you can enable multithreading by setting the option.
+
+```ruby
+set :parallel, true # default is false
+set :n_threads, 4 # default 8
+```
+
+Please notice, due to [GIL](https://en.wikipedia.org/wiki/Global_Interpreter_Lock), we are not able to take advantage of multicore system with parallel processing. But when the threads are block by heavy I/O, which is very common under this circumstance, then it can make considerable performance boost.
+
+### benchmark
+
+Sometimes you just want to know: **why is my script so slow?** In order to efficiently spot the bottleneck of your script, you can enable `benchmark mode`.
+
+```ruby
+set :benchmark, true
+```
+
+Then run your script, you will see the detailed benchmark information.
+
+### hooks
+
+Do something before or/and after the process, define hook blocks.
+
+```ruby
+pre_build do
+  # do something
+end
+
+post_build do
+  # do something
+end
+```
+
+It is pretty self explanatory.
 
 ## Development
 

data/lib/datacraft.rb

@@ -3,8 +3,10 @@ require 'datacraft/exceptions'
 require 'datacraft/definition'
 require 'datacraft/context'
 require 'datacraft/instruction'
+require 'datacraft/parser'
 require 'datacraft/runner'
 
 module Datacraft
   extend Datacraft::Runner
+  extend Datacraft::Parser
 end

data/lib/datacraft/cli.rb

@@ -5,16 +5,17 @@ module Datacraft
   class Cli < Thor
     desc 'build [INSTRUCTION_FILE]', 'build the data by instruction'
     def build(filename)
-      instruction = Instruction.from_file filename
-      Datacraft.run instruction
+      Datacraft.run check(filename)
     end
 
     desc 'check [INSTRUCTION_FILE]',
          'evaluate the instruction without running it'
     def check(filename)
       begin
-        Instruction.from_file filename
+        script = IO.read(filename)
+        instruction = Datacraft.parse script
         puts 'You are ready to go.'
+        instruction
       rescue InvalidInstruction => e
         puts e
       end

data/lib/datacraft/instruction.rb

@@ -33,13 +33,5 @@ module Datacraft
     def set(key, value)
       options[key.to_sym] = value
     end
-
-    def self.from_file(filename)
-      script_content = IO.read(filename)
-      instruction = Instruction.new
-      instruction.instance_eval(script_content)
-      instruction.validate
-      instruction
-    end
   end
 end

data/lib/datacraft/parser.rb

@@ -0,0 +1,14 @@
+module Datacraft
+  module Parser
+    def parse(script = nil, &block)
+      instruction = Instruction.new
+      if script
+        instruction.instance_eval(script)
+      else
+        instruction.instance_eval(&block)
+      end
+      instruction.validate
+      instruction
+    end
+  end
+end

data/lib/datacraft/runner.rb

@@ -34,6 +34,8 @@ module Datacraft
       report measurements if @inst.options[:benchmark]
     end
 
+    private
+
     # output benchmark results
     def report(measurements)
       width = measurements.max_by { |m| m.label.size }.label.size + 1

data/lib/datacraft/version.rb

@@ -1,3 +1,3 @@
 module Datacraft
-  VERSION = '0.4.0'
+  VERSION = '0.4.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datacraft
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Xiaoxing Hu
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-08-15 00:00:00.000000000 Z
+date: 2015-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -122,6 +122,7 @@ files:
 - lib/datacraft/definition.rb
 - lib/datacraft/exceptions.rb
 - lib/datacraft/instruction.rb
+- lib/datacraft/parser.rb
 - lib/datacraft/runner.rb
 - lib/datacraft/version.rb
 homepage: https://github.com/xiaoxinghu/datacraft