beetle_etl 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ebb61022b0e58e217f1b215486a993c7e43799a2
-  data.tar.gz: e00ad7086cf5be7c4cadf520cbd21c1ebd47202d
+  metadata.gz: b08e51fe6819079c6b8317636030d5d678b102cc
+  data.tar.gz: 6afa19db3171f105fa275f99c82c07306aa9c568
 SHA512:
-  metadata.gz: 8e3c2be8adf3cb65807fddb95d1b09d1de9064f586f3e0aefa6026ff3e7b572b8f9653c31ca759114228e6cb6470495f21b74f7470ad6fb7745136210b87f31e
-  data.tar.gz: 3d91601c914486564b8db8e3afdef62c90b04b54fe3633321547de3ec437ee1f12ead442265e84561c4578d2dae482be2cebc85dfd610c76e3db877239d4a2d5
+  metadata.gz: a915efc65e4450aa4ba7cec71a75b89082d6b65be4cd2e6db88ee8e540f9bc0d1db96afd18c787ac504309460447bfdbb20ab462e012ac503e8fe8cdecb65880
+  data.tar.gz: afc74720f9875d7a447030bb14fbc4da90c1004b5c725fb55b0630bbc073d3d15560e6c5e2b7002951e1989eb4b63b956508411811c146986820aee2679a6462

data/.travis.yml

@@ -5,7 +5,7 @@ rvm:
 addons:
   postgresql: "9.3"
   code_climate:
-    repo_token: dd18697b0acb6be343db62982b753b72676e8342701cc0442121de2d12ee6549
+    repo_token: fcd6d8c28da900609a2cf903716d858621b8ce68152edbcebe6908a9a3f5d3d5
 
 before_script:
   - psql -c 'create database travis_ci_test;' -U postgres

data/beetle_etl.gemspec

@@ -19,11 +19,10 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_runtime_dependency 'sequel', '>= 4.13.0'
-  spec.add_runtime_dependency 'celluloid', '>= 0.15.2'
 
   spec.add_development_dependency 'bundler', '~> 1.6'
   spec.add_development_dependency 'rake'
-  spec.add_development_dependency 'rspec', '~> 3.0.0'
+  spec.add_development_dependency 'rspec', '~> 3.1.0'
   spec.add_development_dependency 'pg'
   spec.add_development_dependency 'codeclimate-test-reporter'
   spec.add_development_dependency 'activesupport'

data/lib/beetle_etl/import.rb

@@ -4,28 +4,28 @@ module BeetleETL
     extend self
 
     def run
-      TaskRunner.run(data_steps)
+      TaskRunner.new(data_steps).run
       BeetleETL.database.transaction do
-        load_steps.each(&:run)
+        TaskRunner.new(load_steps).run
       end
     end
 
     private
 
     def data_steps
-      transformations.map do |t|
+      transformations.flat_map do |t|
         [
           Transform.new(t.table_name, t.dependencies, t.query),
           MapRelations.new(t.table_name, t.relations),
           TableDiff.new(t.table_name),
           AssignIds.new(t.table_name),
         ]
-      end.flatten
+      end
     end
 
     def load_steps
       transformations.map do |t|
-        Load.new(t.table_name)
+        Load.new(t.table_name, t.relations)
       end
     end
 

data/lib/beetle_etl/steps/assign_ids.rb

@@ -6,8 +6,10 @@ module BeetleETL
     end
 
     def run
-      assign_new_ids
-      map_existing_ids
+      [
+        Thread.new { assign_new_ids },
+        Thread.new { map_existing_ids }
+      ].each(&:join)
     end
 
     def assign_new_ids

data/lib/beetle_etl/steps/load.rb

@@ -7,10 +7,21 @@ module BeetleETL
       transition
     ]
 
+    attr_reader :relations
+
+    def initialize(table_name, relations)
+      super(table_name)
+      @relations = relations
+    end
+
     def run
-      %w(create update delete undelete).each do |transition|
-        public_send(:"load_#{transition}")
-      end
+      %w(create update delete undelete).map do |transition|
+        Thread.new { public_send(:"load_#{transition}") }
+      end.each(&:join)
+    end
+
+    def dependencies
+      relations.values.map { |d| Load.step_name(d) }.to_set
     end
 
     def load_create

data/lib/beetle_etl/steps/table_diff.rb

@@ -12,9 +12,9 @@ module BeetleETL
     end
 
     def run
-      %w(create keep update delete undelete).each do |transition|
-        public_send(:"transition_#{transition}")
-      end
+      %w(create keep update delete undelete).map do |transition|
+        Thread.new { public_send(:"transition_#{transition}") }
+      end.each(&:join)
     end
 
     def transition_create

data/lib/beetle_etl/task_runner/dependency_resolver.rb

@@ -21,7 +21,7 @@ module BeetleETL
       items = @items.dup
       resolved = []
 
-      while not items.empty?
+      until items.empty?
         resolved_names = resolved.flatten.map(&:name).to_set
 
         resolvable = items.select do |item|

data/lib/beetle_etl/task_runner/task_runner.rb

@@ -1,63 +1,70 @@
-require 'celluloid/autostart'
-
 module BeetleETL
   class TaskRunner
 
-    include Celluloid
+    def initialize(tasks)
+      @dependency_resolver = DependencyResolver.new(tasks)
+      @tasks = tasks
 
-    def initialize(runnables)
-      @runnables = runnables
+      @queue = Queue.new
       @completed = Set.new
       @running = Set.new
-      @dependency_resolver = DependencyResolver.new(runnables)
-
-      run_next
     end
 
-    def completed(runnable_name)
-      @running.delete(runnable_name)
-      @completed << runnable_name
-
-      run_next
-    end
+    def run
+      results = {}
 
-    def run_next
-      if all_run?
-        terminate
-      else
-        resolvables.each do |runnable|
-          unless @running.include?(runnable.name)
-            Task.new(Actor.current, runnable).async.run_task
-            @running << runnable.name
-          end
+      until all_tasks_complete?
+        runnables.each do |task|
+          run_task_async(task)
+          mark_task_running(task.name)
         end
+
+        task_name, task_data = @queue.pop
+        results[task_name] = task_data
+        mark_task_completed(task_name)
       end
+
+      results
     end
 
     private
 
-    def resolvables
-      @dependency_resolver.resolvables(@completed)
+    attr_reader :running, :completed
+
+    def run_task_async(task)
+      Thread.new do
+        started_at = now
+        result = task.run
+        finished_at = now
+
+        @queue.push [task.name, {
+          started_at: started_at,
+          finished_at: finished_at,
+          result: result,
+        }]
+      end
     end
 
-    def all_run?
-      @completed == @runnables.map(&:name).to_set
+    def mark_task_running(task_name)
+      running.add(task_name)
     end
 
-    class Task
+    def mark_task_completed(task_name)
+      runnables.delete(task_name)
+      completed.add(task_name)
+    end
 
-      include Celluloid
+    def runnables
+      resolvables = @dependency_resolver.resolvables(completed)
+      resolvables.reject { |r| running.include? r.name }
+    end
 
-      def initialize(runner, task)
-        @runner = runner
-        @task = task
-      end
+    def all_tasks_complete?
+      @tasks.map(&:name).to_set == completed.to_set
+    end
 
-      def run_task
-        @task.run
-        @runner.async.completed(@task.name)
-        terminate
-      end
+    def now
+      Time.now
     end
 
   end

data/lib/beetle_etl/version.rb

@@ -1,3 +1,3 @@
 module BeetleETL
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/spec/steps/load_spec.rb

@@ -13,7 +13,7 @@ module BeetleETL
     let(:now) { Time.now.beginning_of_day }
     let(:yesterday) { 1.day.ago.beginning_of_day }
 
-    subject { Load.new(:example_table) }
+    subject { Load.new(:example_table, []) }
 
     before do
       BeetleETL.configure do |config|
@@ -52,6 +52,22 @@ module BeetleETL
       end
     end
 
+    describe '#depenencies' do
+      it 'depends on Transform of the same table and AssignIds of its dependees' do
+        relations = {
+          dependee_a_id: :dependee_a,
+          dependee_b_id: :dependee_b,
+        }
+
+        expect(Load.new(:depender, relations).dependencies).to eql(
+          [
+            'dependee_a: Load',
+            'dependee_b: Load',
+          ].to_set
+        )
+      end
+    end
+
     describe '#run' do
       it 'runs all load steps' do
         %w(create update delete undelete).each do |transition|

data/spec/steps/step_spec.rb

@@ -30,7 +30,6 @@ module BeetleETL
       it 'raises an exception' do
         expect { subject.dependencies }.to raise_error(DependenciesNotDefinedError)
       end
-
     end
 
   end

metadata

@@ -1,125 +1,111 @@
 --- !ruby/object:Gem::Specification
 name: beetle_etl
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Luciano Maiwald
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-26 00:00:00.000000000 Z
+date: 2014-12-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 4.13.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 4.13.0
-- !ruby/object:Gem::Dependency
-  name: celluloid
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 0.15.2
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 0.15.2
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 3.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 3.1.0
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: codeclimate-test-reporter
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Taking care of synchronizing external data with referential data in your
@@ -130,8 +116,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -183,17 +169,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.1.11
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: BeetleETL helps you with your recurring ETL imports.