beetle_etl 0.0.2 → 0.0.7

data/spec/steps/transform_spec.rb

@@ -15,6 +15,7 @@ module BeetleETL
           [
             'some_table: Transform',
             'some_other_table: Transform',
+            'example_table: CreateStage',
           ].to_set
         )
       end
@@ -30,5 +31,6 @@ module BeetleETL
         subject.run
       end
     end
+
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: beetle_etl
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.7
 platform: ruby
 authors:
 - Luciano Maiwald
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-31 00:00:00.000000000 Z
+date: 2015-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.13.0
+        version: 4.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.13.0
+        version: 4.0.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -39,75 +39,61 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.6'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 3.1.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 3.1.0
-- !ruby/object:Gem::Dependency
-  name: pg
+  name: timecop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.7.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.7.0
 - !ruby/object:Gem::Dependency
-  name: codeclimate-test-reporter
+  name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.18.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.18.0
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.2.0
 description: Taking care of synchronizing external data with referential data in your
   application.
 email:
@@ -128,8 +114,10 @@ files:
 - lib/beetle_etl/dsl/transformation.rb
 - lib/beetle_etl/dsl/transformation_loader.rb
 - lib/beetle_etl/import.rb
-- lib/beetle_etl/state.rb
+- lib/beetle_etl/naming.rb
 - lib/beetle_etl/steps/assign_ids.rb
+- lib/beetle_etl/steps/create_stage.rb
+- lib/beetle_etl/steps/drop_stage.rb
 - lib/beetle_etl/steps/load.rb
 - lib/beetle_etl/steps/map_relations.rb
 - lib/beetle_etl/steps/step.rb
@@ -140,17 +128,15 @@ files:
 - lib/beetle_etl/version.rb
 - script/postgres
 - spec/beetle_etl_spec.rb
-- spec/dependency_resolver_spec.rb
 - spec/dsl/dsl_spec.rb
 - spec/dsl/transformation_loader_spec.rb
 - spec/dsl/transformation_spec.rb
 - spec/feature/example_schema.rb
 - spec/feature/example_transform.rb
 - spec/feature/feature_spec.rb
-- spec/import_spec.rb
 - spec/spec_helper.rb
-- spec/state_spec.rb
 - spec/steps/assign_ids_spec.rb
+- spec/steps/create_stage_spec.rb
 - spec/steps/load_spec.rb
 - spec/steps/map_relations_spec.rb
 - spec/steps/step_spec.rb
@@ -159,6 +145,7 @@ files:
 - spec/support/database.yml.example
 - spec/support/database.yml.travis
 - spec/support/database_helpers.rb
+- spec/task_runner/dependency_resolver_spec.rb
 homepage: https://github.com/maiwald/beetle_etl
 licenses:
 - MIT
@@ -185,17 +172,15 @@ specification_version: 4
 summary: BeetleETL helps you with your recurring ETL imports.
 test_files:
 - spec/beetle_etl_spec.rb
-- spec/dependency_resolver_spec.rb
 - spec/dsl/dsl_spec.rb
 - spec/dsl/transformation_loader_spec.rb
 - spec/dsl/transformation_spec.rb
 - spec/feature/example_schema.rb
 - spec/feature/example_transform.rb
 - spec/feature/feature_spec.rb
-- spec/import_spec.rb
 - spec/spec_helper.rb
-- spec/state_spec.rb
 - spec/steps/assign_ids_spec.rb
+- spec/steps/create_stage_spec.rb
 - spec/steps/load_spec.rb
 - spec/steps/map_relations_spec.rb
 - spec/steps/step_spec.rb
@@ -204,3 +189,4 @@ test_files:
 - spec/support/database.yml.example
 - spec/support/database.yml.travis
 - spec/support/database_helpers.rb
+- spec/task_runner/dependency_resolver_spec.rb

data/lib/beetle_etl/state.rb

@@ -1,67 +0,0 @@
-module BeetleETL
-
-  ImportAleadyRunning = Class.new(StandardError)
-  ImportSchemaNotFound = Class.new(StandardError)
-  ImportNotRunning = Class.new(StandardError)
-
-  class State
-
-    def start_import
-      raise ImportAleadyRunning if import_already_running?
-
-      @run_id = import_runs_dataset.insert(
-        state: 'RUNNING',
-        started_at: now
-      )
-    end
-
-    def mark_as_succeeded
-      mark_as('SUCCEEDED')
-    end
-
-    def mark_as_failed
-      mark_as('FAILED')
-    end
-
-    def run_id
-      raise ImportNotRunning if @run_id.nil?
-      @run_id
-    end
-
-    def last_run_id
-      last_import = import_runs_dataset.
-        select(:id).
-        where(state: 'SUCCEEDED').
-        order(Sequel.desc(:id)).
-        first
-
-      last_import.nil? ? nil : last_import[:id]
-    end
-
-  private
-
-    def import_runs_table
-      "#{BeetleETL.config.stage_schema}__import_runs".to_sym
-    end
-
-    def import_already_running?
-      import_runs_dataset.where(state: 'RUNNING').count > 0
-    end
-
-    def now
-      Time.now
-    end
-
-    def mark_as(state)
-      import_runs_dataset.filter(id: run_id).update(
-        state: state,
-        finished_at: now
-      )
-    end
-
-    def import_runs_dataset
-      BeetleETL.database[import_runs_table]
-    end
-
-  end
-end

data/spec/import_spec.rb

@@ -1,7 +0,0 @@
-require 'spec_helper'
-
-module BeetleETL
-  describe Import do
-
-  end
-end

data/spec/state_spec.rb

@@ -1,124 +0,0 @@
-require 'spec_helper'
-
-require 'active_support/core_ext/date/calculations'
-require 'active_support/core_ext/numeric/time'
-
-module BeetleETL
-  describe State do
-    subject { State.new }
-
-    before do
-      BeetleETL.configure do |config|
-        config.stage_schema = 'stage'
-        config.database = test_database
-      end
-
-      test_database.create_schema 'stage'
-      test_database.create_table :stage__import_runs do
-        primary_key :id
-        String :state, size: 10, null: false
-        DateTime :started_at, null: false
-        DateTime :finished_at
-      end
-    end
-
-    describe '#start_import' do
-      let(:now) { 1.minute.ago.beginning_of_day }
-
-      it 'registers a new import in the import_runs table' do
-        allow(subject).to receive(:now) { now }
-
-        subject.start_import
-
-        expect(:stage__import_runs).to have_values(
-          [ :id , :state    , :started_at , :finished_at ] ,
-          [ 1   , 'RUNNING' , now         , nil          ]
-        )
-      end
-
-      it 'raises an exception if there is alreay an import marked as running' do
-        insert_into(:stage__import_runs).values(
-          [ :id , :state    , :started_at , :finished_at ] ,
-          [ 1   , 'RUNNING' , now         , nil          ]
-        )
-
-        expect { subject.start_import }.to raise_exception(BeetleETL::ImportAleadyRunning)
-      end
-    end
-
-    context 'run ids' do
-      before do
-        insert_into(:stage__import_runs).values(
-          [ :state      , :started_at , :finished_at ] ,
-          [ 'FAILED'    , 8.days.ago  , 7.days.ago   ] ,
-          [ 'SUCCEEDED' , 6.days.ago  , 5.day.ago    ] ,
-          [ 'SUCCEEDED' , 4.days.ago  , 3.days.ago   ] ,
-          [ 'FAILED'    , 2.days.ago  , 1.day.ago    ] ,
-        )
-      end
-
-      describe '#run_id' do
-        it 'returns the import‘s id after it has been started' do
-          subject.start_import
-          expect(subject.run_id).to eql(5)
-        end
-
-        it 'raises an exception when the import has not been started' do
-          expect { subject.run_id }.to raise_exception(BeetleETL::ImportNotRunning)
-        end
-      end
-
-      describe '#last_run_id' do
-        it 'returns nil if there is no last successful import' do
-          test_database[:stage__import_runs].update(state: 'FAILED')
-
-          subject.start_import
-          expect(subject.last_run_id).to be_nil
-        end
-
-        it 'returns the id of the last successul import' do
-          subject.start_import
-          expect(subject.last_run_id).to eql(3)
-        end
-      end
-    end
-
-    context 'marking imports' do
-      let(:now) { 1.minute.ago.beginning_of_day }
-      let(:one_day_ago) { 1.day.ago.beginning_of_day }
-
-      before do
-        insert_into(:stage__import_runs).values(
-          [ :state      , :started_at , :finished_at ] ,
-          [ 'SUCCEEDED' , 2.days.ago  , one_day_ago  ] ,
-        )
-        allow(subject).to receive(:now) { now }
-        subject.start_import
-      end
-
-      describe '#mark_as_failed' do
-        it 'marks the current import as FAILED' do
-          subject.mark_as_failed
-
-          expect(:stage__import_runs).to have_values(
-            [ :id , :state      , :finished_at ] ,
-            [ 1   , 'SUCCEEDED' , one_day_ago  ] ,
-            [ 2   , 'FAILED'    , now          ] ,
-          )
-        end
-      end
-
-      describe '#mark_as_succeeded' do
-        it 'marks the current import as SUCCEEDED' do
-          subject.mark_as_succeeded
-
-          expect(:stage__import_runs).to have_values(
-            [ :id , :state      , :finished_at ] ,
-            [ 1   , 'SUCCEEDED' , one_day_ago  ] ,
-            [ 2   , 'SUCCEEDED' , now          ] ,
-          )
-        end
-      end
-    end
-  end
-end