tango-etl 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 35dc1dc259634c71c5549cd9f44c03632bd446d6
-  data.tar.gz: 4fb85e9a4017c02480b6194686a16ee6040791f8
+  metadata.gz: 4612c4976d0f3ebed10de7d9a4fdbf8178ab8072
+  data.tar.gz: 47afa4fffe497491fbcecd49ee8b001b88a4f9b6
 SHA512:
-  metadata.gz: c945d4126a292c411ddc7a741363737181a1722849c2b43a50d0526773fe7fd989abc56075174f9b63df2452cb5da0268e808274ff39f597cc6608e4e866410b
-  data.tar.gz: a4ce34f890c40498e894ed8dcf1429daae1597e4bfd05671c9a12182dc589d1d24a1574b5c05b5b02aac029361b99044dc540a53381ae7543db191887414f0df
+  metadata.gz: b487d74cd0dc5e319ef758d52e969727828f240496c345b60731d78663e015c2f5891f2bc405d9da2a9670b0bd0a56688c413c4f183a8c639ad656dd319edcd1
+  data.tar.gz: e9edf5573f63b9723d3b89e91f99390fd38d865f92dddeebc66e496812a3a009515e0a3ef3cd29d0e1213d55ed9f50540540132a36ad3b2f7f67dadd0b27e3b3

data/CHANGELOG.md

@@ -8,4 +8,7 @@
 - Added full API documentation
 
 ###0.1.2 - 30/06/2014
-- Update of gemfile
+- Update of gemfile
+
+###0.1.3 - 29/07/2014
+- Change way of registering tango application dependencies

data/Gemfile

@@ -11,6 +11,7 @@ group :development do
 end
 
 group :test do
+  gem 'minitest', '~> 5.3.5'
   gem 'shoulda-context', '~> 1.2.1'
   gem "mocha", "~> 1.1.0"
   gem 'activerecord-nulldb-adapter', '~> 0.3.1'

data/lib/tango/app.rb

@@ -34,6 +34,9 @@ module Tango
       @parser = parser || Nokogiri::HTML
       @db_locker = db_locker || DatabaseLocker.new( Multidb.databases )
       @logger = logger || Logger.new( STDOUT )
+
+      @models = []
+      @operators = []
       
     end
     
@@ -44,46 +47,40 @@ module Tango
     # Filter run after Tango execution
     def after
     end
-    
-    # Register new resource model
+
+    # Register a new model
     #
-    # @param symbol [Symbol]
-    # @param model [Class]
-    def register_model( symbol, model )
-      
-      @models[symbol] = model
-      
-      # Truncate table of non persistent model
-      unless model.persistent?
-        ActiveRecord::Base.connection.execute( "TRUNCATE #{model.table_name}" )
-      end
-      
+    # @param  model   [Symbol]
+    # @return         [Array]
+    def register_model( model )
+      @models << model    
     end
     
-    # Register new resource operator
+    # Register a new resource operator
     #
-    # @param symbol [Symbol]
-    # @param operator [Class]
-    def register_operator( symbol, operator )
-      
-      @operators[symbol] = operator
-      
-      # Register operator with resource cache system
-      @cache.register( symbol ) do |resource|
-        operator.load( resource )
-      end
-      
+    # @param  operator [Tango::ETL::OperatorInterface]
+    # @return          [Array] 
+    def register_operator( operator )
+      @operators << operator
+    end
+
+    # Register a new handler with the dispatcher   
+    #
+    # @param  handler [Tango::ETL::HandlerInterface]
+    # @return         [Array] 
+    def register_handler( handler )
+      @dispatcher.register( handler )
     end
     
     # Run ETL process
     # 
-    # @param link_stack [Tango::LinkStack]
-    # @param dispatcher [Tango::Etl::Dispatcher]
-    # @param cache [Tango::Resources::Cache]
-    # @param http_client [Object] Must implement get method
-    # @param parser [Object] Must implement parse method
-    # @param logger [Logger]
-    # @return [Integer] 
+    # @param  link_stack  [Tango::LinkStack]
+    # @param  dispatcher  [Tango::Etl::Dispatcher]
+    # @param  cache       [Tango::Resources::Cache]
+    # @param  http_client [Object] Must implement get method
+    # @param  parser      [Object] Must implement parse method
+    # @param  logger      [Logger]
+    # @return             [Nil] 
     def run
     
       # Save beginning time
@@ -93,13 +90,17 @@ module Tango
       @logger.info "Target: #{@link_stack.host}."
       
       # Use next unlocked database
-      Multidb.use( @db_locker.unlocked )
+      pick_database( @db_locker.unlocked )
       @logger.info "Using database '#{@db_locker.unlocked}'."
+
+      @logger.info "Truncating non persistent models ..." 
+      truncate_tables( non_persistent_models )
       
-      # Run before filter
+      # Load cache for persistent models
       @logger.info "Loading cache ..."
-      load_cache
-      
+      setup_cache( @operators )
+      load_cache( persistent_models )
+
       # Run before filter
       @logger.info "Running before callback ..."
       before
@@ -124,13 +125,14 @@ module Tango
         begin 
           response = @http_client.get( @link_stack.host + link )
         rescue StandardError => e 
-          @logger.error "Could not download contents of #{@link_stack.host + link} link."; @logger.error e.message
+          @logger.error "Could not download contents of #{@link_stack.host + link} link."
+          @logger.error e.message
           next
         end
         
         # Continue only when response has code 200 or 201
         if ! [ 200, 201 ].include?( response.code )
-          @logger.error "Response code for link #{link} is #{response.code}. Only code 200 is accepted."
+          @logger.error "Response code for link #{link} is #{response.code}. Only codes 200 and 201 are accepted."
           next
         end
         
@@ -162,14 +164,14 @@ module Tango
       
       # Release buffers
       @logger.info "Releasing buffers ..."
-      @cache.buffer.release_all()
+      release_buffer( @cache.buffer )
       
       # Run after filter
       @logger.info "Running after callback ..."
       after
       
       # Lock database used in this Tango iteration
-      @db_locker.lock( @db_locker.unlocked )
+      lock_database( @db_locker.unlocked )
     
       # Get time of script execution ending
       end_time = Time.now
@@ -183,15 +185,76 @@ module Tango
     end
     
     private
+
+    # Pick database ( e.g. to be used in this Tango run )
+    #
+    # @param  database  [String|Symbol]
+    # @return           [String|Symbol] 
+    def pick_database( database )
+      Multidb.use( database )
+    end
+
+    # Lock database ( e.g. used in this Tango run )
+    #
+    # @param  database  [String|Symbol]
+    # @return           [String|Symbol] 
+    def lock_database( database )
+      @db_locker.lock( database )
+    end
+
+    # Fetch list of persistent model registered with application
+    # 
+    # @return [Array]
+    def persistent_models
+      @models.select { |m| m.persistent? }
+    end
+
+    # Fetch list of non persistent model registered with application
+    # 
+    # @return [Array]
+    def non_persistent_models
+      @models.reject { |m| m.persistent? }
+    end
+
+    # Truncate table of given models
+    #
+    # @return [Nil]
+    def truncate_tables( models )
+      models.each do |model|
+        ActiveRecord::Base.connection.execute( "TRUNCATE #{model.table_name}" )
+      end
+    end
     
-    def load_cache
-      
-      @models.each do |symbol, model|
-        model.all.each do |m|
-          @cache.set( symbol, m )
-        end if model.persistent?    
+    # Register cache with resource operators
+    #
+    # @param  operators [Array]
+    # @return           [Array]
+    def setup_cache( operators )
+      operators.each do |operator|
+        @cache.register( Tango::Kernel.symbolize( operator ) ) do |resource|
+          operator.load( resource )
+        end
       end
-      
+    end
+
+    # Load cache for given models
+    #
+    # @return [Nil]
+    def load_cache( models )
+      models.each do |model|
+        symbol = Tango::Kernel.symbolize( model ) 
+        model.all.each do |record|
+          @cache.set( symbol, record )
+        end
+      end
+    end
+
+    # Release given buffer
+    #
+    # @param  buffer [Tango::Resource::Buffer]
+    # @return        [Nil]
+    def release_buffer( buffer )
+      buffer.release_all
     end
     
   end

data/lib/tango/etl/handler_interface.rb

@@ -11,7 +11,10 @@ module Tango
       
       # Constructor of Tango's handler 
       #
-      # @param  url  [String]
+      # @param  [String]
+      # @param  [Nokogiri::XML]
+      # @param  [Tango::Resource::Cache]
+      # @return [Tango::ETL::HandlerInterface]
       def initialize( url, document, cache = nil )
         @url = url
         @document = document

data/lib/tango/kernel.rb

@@ -1,25 +1,19 @@
 module Tango
   module Kernel
     
-    # Convert file path to class name
-    # @param  file_path [String]
-    # @return           [String]
-    def self.classify( file_path )
-      File.basename( file_path, ".*" ).split( "_" ).map { |w| w.capitalize }.join
-    end
 
-    # Load class from a file
-    # 
-    # @param  file          [String]
-    # @param  module_prefix [String]
-    # @return               [Class]
-    def self.load( file, module_prefix = "" )  
-        
-        require file
-        
-        class_name = Kernel.classify( file )
-        Kernel.const_get( "#{module_prefix}#{class_name}" )
-        
+    # Fetch list of first level classes in module
+    #
+    # @param  mod   [Module]
+    # @return       [Array]
+    def self.module_classes( mod )
+
+      mod.constants.map { |s|
+        mod.const_get( s )
+      }.select { |c|  
+        Class === c
+      }
+
     end
     
     # Obtain symbol of a class

data/lib/tango/resource/buffer.rb

@@ -42,12 +42,11 @@ module Tango
       def fill( type, resource )
         
         raise ArgumentError, "Trying to fill object with unregistered type" unless @resources.keys.include?( type )
-    
+
         # Append resource to the buffer
         @resources[type] << resource
         # Release the buffer if buffer size exceeded
-        release( type ) if @resources[type].count >= @size
-        
+        release( type ) if @resources[type].count >= @size   
       end
       
       # Release all registered buffers

data/lib/tango/resource/cache.rb

@@ -45,6 +45,7 @@ module Tango
         # Get resource from cache
         cached_resource = get( type, resource )
 
+
         unless cached_resource
           
           raise ArgumentError, "No resource callback given" unless block_given?

data/lib/tango/version.rb

@@ -1,3 +1,3 @@
 module Tango  
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

data/test/support/lib/classes.rb

@@ -0,0 +1,25 @@
+module Foo
+	
+	class A
+	end
+
+	class B
+	end
+
+	class C
+	end
+
+	module Bar
+		
+		class A
+		end
+
+		class B
+		end
+
+		class C
+		end
+		
+	end
+
+end

data/test/unit/etl/test_dispatcher.rb

@@ -1,4 +1,4 @@
-require_relative '../../test_helper'
+require_relative '../../support/helper'
 require_relative '../../support/lib/simple_handler.rb'
 
 require 'tango/etl/dispatcher'

data/test/unit/resource/test_buffer.rb

@@ -1,4 +1,4 @@
-require_relative '../../test_helper'
+require_relative '../../support/helper'
 
 require 'tango/resource/buffer'
 

data/test/unit/resource/test_cache.rb

@@ -1,5 +1,5 @@
-require_relative '../../test_helper'
-require_relative '../../support/lib/simple_buffer.rb'
+require_relative '../../support/helper'
+require_relative '../../support/lib/simple_buffer'
 
 require 'tango/resource/cache'
 

data/test/unit/test_abstract_model.rb

@@ -1,4 +1,4 @@
-require_relative '../test_helper.rb'
+require_relative '../support/helper'
 
 require 'active_record'
 require 'activerecord-nulldb-adapter'

data/test/unit/test_database_locker.rb

@@ -1,4 +1,4 @@
-require_relative '../test_helper.rb'
+require_relative '../support/helper'
 
 require 'tango/database_locker'
 

data/test/unit/test_kernel.rb

@@ -1,4 +1,4 @@
-require_relative '../test_helper'
+require_relative '../support/helper'
 
 require 'tango/kernel'
 require 'tango/abstract_model.rb'
@@ -8,21 +8,17 @@ class TestKernel < Test::Unit::TestCase
   context "a kernel" do
     
     setup do
+      
       @lib_path = File.join( File.expand_path( '../../', __FILE__ ), 'support', 'lib' )
+      
+      Dir.glob( File.join( @lib_path,  '*.rb' ) ) do |f|
+        require f
+      end
+      
     end
 
-    should "transform file path to name of a class" do
-      assert_equal "FooBar", Tango::Kernel.classify( File.join( @lib_path, 'foo_bar.rb' ) )
-    end
-    
-    should "load a class from a file" do
-      klass = Tango::Kernel.load( File.join( @lib_path, 'simple_buffer.rb' ) )
-      assert_equal SimpleBuffer, klass
-    end
-    
-    should "load a class in a module from a file" do
-      klass = Tango::Kernel.load( File.join( @lib_path, 'user.rb' ), 'Model::' )
-      assert_equal Model::User, klass
+    should "load classes in module" do
+      assert_equal [Foo::A, Foo::B, Foo::C], Tango::Kernel.module_classes( Foo )
     end
     
     should "obtain symbol from a class" do

data/test/unit/test_link_stack.rb

@@ -1,5 +1,4 @@
-require_relative '../test_helper.rb'
-
+require_relative '../support/helper'
 require 'tango/link_stack'
 
 class TestLinkStack < Test::Unit::TestCase

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tango-etl
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Maciej Komorowski
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-30 00:00:00.000000000 Z
+date: 2014-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -164,12 +164,13 @@ files:
 - lib/tango/resource/buffer.rb
 - lib/tango/resource/cache.rb
 - lib/tango/version.rb
-- tango.gemspec
+- tango-etl.gemspec
 - test/support/db/schema.rb
+- test/support/helper.rb
+- test/support/lib/classes.rb
 - test/support/lib/simple_buffer.rb
 - test/support/lib/simple_handler.rb
 - test/support/lib/user.rb
-- test/test_helper.rb
 - test/unit/etl/test_dispatcher.rb
 - test/unit/resource/test_buffer.rb
 - test/unit/resource/test_cache.rb
@@ -201,6 +202,5 @@ rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: ETL framework
-test_files:
-- test/test_helper.rb
+test_files: []
 has_rdoc: