chicago-etl 0.2.7 → 0.3.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZjJjZDBlOTdlMDlmZTZhNzAyM2Q1N2YxZTkxY2Q0YjA2ZThhNGU2OA==
+    NmMzMzcxMGE2ZjhmOTU5OTAyNmIzMDNiYjdmY2E5OGRjYTdiNDExYw==
   data.tar.gz: !binary |-
-    NWY2M2Q1ZWUyMTFlNzY0NGM3MTBhMDM1MjZiMjlmYjI4OTdhNjM3MA==
+    OGM1OGM3NzBmMjA3OTA5NWQxNjcxMDQ3NDIzMmNkY2ZkYzI4MTBlNg==
 SHA512:
   metadata.gz: !binary |-
-    YzA1MzI3MGFlMjc1ZmMyODc3MjJkODlhZDRjNzU1ODQ4MTdmNDhhZjFhYzg3
-    N2Q3NDY4ZmI2NmI1OTg1YTdkNjBmYjQxOGVjNmYwMzEyMTg0YzZiODA5MmQ2
-    NjcxZTk1NzliMWI3MWZlMWQyM2M5NjFmMzU1NjNhMjhmYThhMTY=
+    MjhhODU2MDU0MmZmM2NmMzczMDEyZGI2MmVjMDAzYTI3MTEzYjZjMjZhOTI5
+    ODM2Zjc2YzYyY2Q3MGUzZWI4NjYxYzQ0ZDA1NGY2OThmNWM1OTBhMjI0N2Nj
+    NDIwYjRjMGI1YzI4NmJkNGIzOTRmMzk5OTk3YTY4ZmM2MDQ5NzM=
   data.tar.gz: !binary |-
-    M2Y4ZDY0NjVhY2U1NTEwZmQxOGYzMjM5NGViMTE2MWIyZDIxZGYzOTU3ZGE0
-    NTkyODU5YWMzMzkwNTMxODg0ZWRkOGZiZDUyMWNmNzVlOWYxMTBlOTM3NWYx
-    NDkzNzdmYzQyY2E0MWI4NzdhZGRlNzdlYTYyYTgzOWFiYzBkOWE=
+    N2RlM2JlYjc0MTRjMmRlMjhkMDUyYTlhZWZjMzU2YTA0YWI1M2FkYzFmNjEz
+    YzU5MDE1MTJjNGU4Mzc4ZWQzNzIxYzM3OTI5NzA3YTAzNTQ0ODhkODU2Njc1
+    ZTZmMWUyOWRiYjNhNGI5ZGM4YzFlZmNmYzRiZWQ0MWIwNmNkMTQ=

data/VERSION

@@ -1 +1 @@
-0.2.7
+0.3.0

data/chicago-etl.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: chicago-etl 0.2.7 ruby lib
+# stub: chicago-etl 0.3.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "chicago-etl"
-  s.version = "0.2.7"
+  s.version = "0.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Roland Swingler"]
-  s.date = "2014-09-01"
+  s.date = "2014-09-08"
   s.description = "ETL tools for Chicago"
   s.email = "roland.swingler@gmail.com"
   s.extra_rdoc_files = [
@@ -27,7 +27,6 @@ Gem::Specification.new do |s|
     "Rakefile",
     "VERSION",
     "chicago-etl.gemspec",
-    "chicago-flow.gemspec",
     "lib/chicago-etl.rb",
     "lib/chicago/etl.rb",
     "lib/chicago/etl/array_sink.rb",

data/lib/chicago/etl/batch.rb

@@ -22,7 +22,11 @@ module Chicago
         #
         # This should be used in preference to new or create.
         def instance
-          (last_batch.nil? || last_batch.finished?) ? new : last_batch
+          if last_batch.nil? || last_batch.finished? || last_batch.started_at.to_date < Date.today
+            new
+          else 
+            last_batch
+          end
         end
       
         # Returns the last batch run, or nil if this is the first batch.
@@ -57,6 +61,21 @@ module Chicago
         !!@reextract
       end
 
+      def extract_from
+        return if reextracting?
+        value = self.class.dataset.
+          where(:state => "Finished").
+          where {|r| r.id < id }.
+          select(:max.sql_function(:extracted_to).as(:extracted_to)).
+          single_value
+
+        if value && value.to_date == Date.today
+          (value.to_date - 1).to_time
+        else
+          value
+        end
+      end
+
       # Deprecated.
       #
       # @deprecated Use perform_task instead

data/spec/etl/batch_spec.rb

@@ -1,4 +1,5 @@
 require 'spec_helper'
+require 'date'
 
 describe Chicago::ETL::Batch do
   before :each do
@@ -55,15 +56,86 @@ describe Chicago::ETL::Batch do
     batch.should be_in_error
   end
 
-  it "should not return a new batch if the last batch was not finished" do
-    batch = ETL::Batch.instance.start
-    ETL::Batch.instance == batch
+  it "returns nil from extract_from when re-extracting" do
+    batch = ETL::Batch.instance
+    batch.reextract
+    batch.start
+    expect(batch.extract_from).to be_nil
   end
 
-  it "should not return a new batch if the last batch ended in error" do
-    batch = ETL::Batch.instance.start
-    batch.error
-    ETL::Batch.instance.should == batch
+  it "returns nil from extract_from when the first batch" do
+    batch = ETL::Batch.instance
+    batch.start
+    expect(batch.extract_from).to be_nil
+  end
+
+  it "returns the previous finised batch's extracted_to as extract_from" do
+    Timecop.freeze(2014, 01, 6, 0, 0, 0) {
+      ETL::Batch.new.start.finish
+    }
+
+    Timecop.freeze(2014, 01, 10, 0, 0, 0) {
+      ETL::Batch.new.start.finish
+    }
+
+    ETL::Batch.new.start.error
+    
+    batch = ETL::Batch.new.start
+    expect(batch.extract_from).to eql(Time.local(2014,1,10,0,0,0))
+  end
+
+  it "returns the previous finised batch's extracted_to as extract_from" do
+    Timecop.freeze(2014, 01, 6, 0, 0, 0) {
+      ETL::Batch.new.start.finish
+    }
+
+    Timecop.freeze(2014, 01, 8, 0, 0, 0) {
+      ETL::Batch.new.start.error
+    }
+
+    Timecop.freeze(2014, 01, 10, 0, 0, 0) {
+      ETL::Batch.new.start.finish
+    }
+    
+    batch = ETL::Batch.new.start
+    expect(batch.extract_from).to eql(Time.local(2014,1,10,0,0,0))
+  end
+
+  it "returns yesterday, rather than extract_from if extract_from is today" do
+    Timecop.freeze(2014, 01, 6, 0, 0, 0)
+
+    ETL::Batch.new.start.finish
+    
+    batch = ETL::Batch.new.start
+    expect(batch.extract_from).to eql(Time.local(2014,1,5,0,0,0))
+  end
+
+  context "when rerun in the same day" do
+    it "should not return a new batch if the last batch was not finished" do
+      batch = ETL::Batch.instance.start
+      expect(ETL::Batch.instance).to eql(batch)
+    end
+
+    it "should not return a new batch if the last batch ended in error" do
+      batch = ETL::Batch.instance.start
+      batch.error
+      ETL::Batch.instance.should == batch
+    end
+  end
+
+  context "when rerun a day later" do
+    it "returns a new batch when the previous batch was unfinished" do
+      batch = ETL::Batch.instance.start
+      Timecop.freeze(Date.today + 1)
+      expect(ETL::Batch.instance).to_not eql(batch)
+    end
+
+    it "returns a new batch when the previous batch was in error" do
+      batch = ETL::Batch.instance.start
+      batch.error
+      Timecop.freeze(Date.today + 1)
+      expect(ETL::Batch.instance).to_not eql(batch)
+    end
   end
 
   it "should create a log in tmp/batches/1/log" do

data/spec/spec_helper.rb

@@ -22,4 +22,7 @@ end
 Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
 
 RSpec.configure do |config|
+  config.after :each do
+    Timecop.return
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chicago-etl
 version: !ruby/object:Gem::Version
-  version: 0.2.7
+  version: 0.3.0
 platform: ruby
 authors:
 - Roland Swingler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-01 00:00:00.000000000 Z
+date: 2014-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chicagowarehouse
@@ -200,7 +200,6 @@ files:
 - Rakefile
 - VERSION
 - chicago-etl.gemspec
-- chicago-flow.gemspec
 - lib/chicago-etl.rb
 - lib/chicago/etl.rb
 - lib/chicago/etl/array_sink.rb

data/chicago-flow.gemspec

@@ -1,92 +0,0 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
-# -*- encoding: utf-8 -*-
-
-Gem::Specification.new do |s|
-  s.name = "chicago-flow"
-  s.version = "0.0.1"
-
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Roland Swingler"]
-  s.date = "2013-06-05"
-  s.description = "Dataflow-style processing for hash-like rows"
-  s.email = "roland.swingler@gmail.com"
-  s.extra_rdoc_files = [
-    "LICENSE.txt",
-    "README.rdoc"
-  ]
-  s.files = [
-    ".document",
-    ".rspec",
-    "Gemfile",
-    "LICENSE.txt",
-    "README.rdoc",
-    "Rakefile",
-    "VERSION",
-    "lib/chicago/flow.rb",
-    "lib/chicago/flow/array_sink.rb",
-    "lib/chicago/flow/array_source.rb",
-    "lib/chicago/flow/dataset_source.rb",
-    "lib/chicago/flow/filter.rb",
-    "lib/chicago/flow/mysql.rb",
-    "lib/chicago/flow/mysql_file_serializer.rb",
-    "lib/chicago/flow/mysql_file_sink.rb",
-    "lib/chicago/flow/pipeline_endpoint.rb",
-    "lib/chicago/flow/pipeline_stage.rb",
-    "lib/chicago/flow/sink.rb",
-    "lib/chicago/flow/transformation.rb",
-    "lib/chicago/flow/transformation_chain.rb",
-    "spec/array_sink_spec.rb",
-    "spec/array_source_spec.rb",
-    "spec/database.yml.dist",
-    "spec/dataset_source_spec.rb",
-    "spec/filter_spec.rb",
-    "spec/mysql_file_serializer_spec.rb",
-    "spec/mysql_file_sink_spec.rb",
-    "spec/mysql_integration_spec.rb",
-    "spec/pipeline_stage_spec.rb",
-    "spec/spec_helper.rb",
-    "spec/transformation_chain_spec.rb",
-    "spec/transformation_spec.rb"
-  ]
-  s.homepage = "http://github.com/notonthehighstreet/chicago-flow"
-  s.licenses = ["MIT"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.24"
-  s.summary = "Dataflow-style processing for hash-like rows"
-
-  if s.respond_to? :specification_version then
-    s.specification_version = 3
-
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<fastercsv>, [">= 0"])
-      s.add_runtime_dependency(%q<sequel>, [">= 0"])
-      s.add_runtime_dependency(%q<sequel_load_data_infile>, [">= 0.0.2"])
-      s.add_runtime_dependency(%q<sequel_fast_columns>, [">= 0"])
-      s.add_development_dependency(%q<mysql>, ["= 2.8.1"])
-      s.add_development_dependency(%q<rspec>, ["~> 2"])
-      s.add_development_dependency(%q<bundler>, ["~> 1"])
-      s.add_development_dependency(%q<jeweler>, ["~> 1.8.4"])
-    else
-      s.add_dependency(%q<fastercsv>, [">= 0"])
-      s.add_dependency(%q<sequel>, [">= 0"])
-      s.add_dependency(%q<sequel_load_data_infile>, [">= 0.0.2"])
-      s.add_dependency(%q<sequel_fast_columns>, [">= 0"])
-      s.add_dependency(%q<mysql>, ["= 2.8.1"])
-      s.add_dependency(%q<rspec>, ["~> 2"])
-      s.add_dependency(%q<bundler>, ["~> 1"])
-      s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
-    end
-  else
-    s.add_dependency(%q<fastercsv>, [">= 0"])
-    s.add_dependency(%q<sequel>, [">= 0"])
-    s.add_dependency(%q<sequel_load_data_infile>, [">= 0.0.2"])
-    s.add_dependency(%q<sequel_fast_columns>, [">= 0"])
-    s.add_dependency(%q<mysql>, ["= 2.8.1"])
-    s.add_dependency(%q<rspec>, ["~> 2"])
-    s.add_dependency(%q<bundler>, ["~> 1"])
-    s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
-  end
-end
-