tango-etl 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0ca6376b79e444981adcf8ee8c66f4cd2a67d6d5
+  data.tar.gz: 0e567db2d98e6e1d7fc70c308e26d0959ab82541
+SHA512:
+  metadata.gz: 8d51e56712c4aecaca7bd5d1b67e9fad5b5846ed888d83a4cbc6265364c0e7f69dcf8a7f2cb0513bad4324c6ac003125ed7e56a0416f5cf9cd543a1c1a6b96eb
+  data.tar.gz: 7ec825c8bc994fd1d0d765d6345e3c39f8fb2f75bc4559663816ebd0cf5aefe0be6df2ec7ee397ba186336bd0810577500bc64d6e3292cbd828735f35c92ef50

data/Gemfile

@@ -0,0 +1,19 @@
+source 'https://rubygems.org'
+
+ruby '2.0.0'
+
+gem 'nokogiri', '~> 1.6.1'
+gem 'httparty', '~> 0.13.1'
+gem 'activerecord', '~> 4.1.0'
+gem 'activerecord-import', '~> 0.5.0'
+gem 'ar-multidb', '~> 0.1.12'
+
+group :development do
+  gem "yard", "~> 0.8.7"
+end
+
+group :test do
+  gem "shoulda", "~> 3.5.0"
+  gem "mocha", "~> 1.0.0"
+  gem 'activerecord-nulldb-adapter', '~> 0.3.1'
+end

data/Rakefile

@@ -0,0 +1,12 @@
+require 'rake/testtask' 
+
+# By default go with run task
+task :default => "test:unit"
+
+namespace :test do
+  
+  Rake::TestTask.new( :unit ) do |t|
+    t.test_files = FileList[ 'test/unit/test_*.rb', 'test/unit/*/test_*.rb' ]
+  end
+  
+end

data/changelog.md

@@ -0,0 +1,4 @@
+#Changelog
+
+###0.1.0 - April 2014
+- Tango separated from [UEK-Tango](https://bitbucket.org/Mckomo/uek-tango/) as an independent gem

data/config/app.yml.sample

@@ -0,0 +1,6 @@
+production:
+  site_index: http://example.com/
+  sleep: 0.5
+development:
+  site_index: http://localhost:8000/
+  sleep: 0

data/config/database.yml.sample

@@ -0,0 +1,27 @@
+development:
+  adapter: mysql2
+  database: tango_dev_master
+  username: root
+  password: pass
+  host: localhost
+  multidb:
+    databases:
+      slave:
+        database: tango_dev_slave
+production:
+  adapter: mysql2
+  database: tango_prod_master
+  username: root
+  password: pass
+  host: localhost
+  multidb:
+    databases:
+      slave:
+        database: tango_prod_slave
+test:
+  adapter: mysql2
+  database: tango_test
+  username: root
+  password: pass
+  host: localhost
+ 

data/lib/tango/abstract_model.rb

@@ -0,0 +1,53 @@
+module Tango
+  
+  # Base model for Tango resources
+  # 
+  # @author Mckomo
+  class AbstractModel < ::ActiveRecord::Base
+        
+    # Required by ActiveRecord
+    self.abstract_class = true
+    
+    @properties = nil
+    @last_id = nil
+    
+    # Return array with values of model properties
+    #
+    # @return [Array]
+    def values
+      self.attributes.values
+    end
+    
+    # Return cache key of model instance
+    #
+    # @return [Object]
+    def cache_key
+      raise NotImplementedError
+    end
+    
+    # Return array with names of model properties
+    #
+    # @return [Array]
+    def self.properties
+      @properties || @properties = self.attribute_names.map { |a| a.to_sym }
+    end
+    
+    # Return incremented value of last id in the model's table
+    #
+    # @return [Integer]
+    def self.next_id
+      @last_id ||= self.pluck( :id ).last || 0
+      @last_id += 1
+    end
+    
+    # State wether model should be cached or not
+    #
+    # @return [Boolean]
+    def self.persistent?
+      raise NotImplementedError
+    end
+    
+  end
+  
+end
+

data/lib/tango/app.rb

@@ -0,0 +1,202 @@
+require 'tango/version'
+
+module Tango
+  
+  # Interface for Tango app runtime filters
+  #
+  # @author Mckomo
+  class App
+    
+    attr_reader :config, :dispatcher, :link_stack, :logger
+    
+    # @param link_stack   [Tango::LinkStack]
+    # @param dispatcher   [Tango::Etl::Dispatcher]
+    # @param cache        [Tango::Resources::Cache]
+    # @param http_client  [Object] Must implement get method
+    # @param parser       [Object] Must implement parse method
+    # @param db_locker    [DatabaseLocker]
+    # @param logger       [Logger]
+    # @return             [Tango::App]
+    def initialize( config: {}, link_stack: nil, dispatcher: nil, cache: nil, http_client: nil, parser: nil, db_locker: nil, logger: nil )
+      
+      # Init app properties
+      @models = {}
+      @operators = {}
+      
+      # Set config
+      @config = config
+      
+      # Set dependencies
+      @link_stack = link_stack || LinkStack.new( config['target_url'] )
+      @dispatcher = dispatcher || ETL::Dispatcher.new
+      @cache = cache || Resource::Cache.new( Resource::Buffer.new )
+      @http_client = http_client || HTTParty
+      @parser = parser || Nokogiri::HTML
+      @db_locker = db_locker || DatabaseLocker.new( Multidb.databases )
+      @logger = logger || Logger.new( STDOUT )
+      
+    end
+    
+    # Filter run before Tango execution
+    def before
+      raise NotImplementedError
+    end
+    
+    # Filter run after Tango execution
+    def after
+      raise NotImplementedError
+    end
+    
+    # Register new resource model
+    #
+    # @param symbol [Symbol]
+    # @param model [Class]
+    def register_model( symbol, model )
+      
+      @models[symbol] = model
+      
+      # Truncate table of non persistent model
+      unless model.persistent?
+        ActiveRecord::Base.connection.execute( "TRUNCATE #{model.table_name}" )
+      end
+      
+    end
+    
+    # Register new resource operator
+    #
+    # @param symbol [Symbol]
+    # @param operator [Class]
+    def register_operator( symbol, operator )
+      
+      @operators[symbol] = operator
+      
+      # Register operator with resource cache system
+      @cache.register( symbol ) do |resource|
+        operator.load( resource )
+      end
+      
+    end
+    
+    # Run ETL process
+    # 
+    # @param link_stack [Tango::LinkStack]
+    # @param dispatcher [Tango::Etl::Dispatcher]
+    # @param cache [Tango::Resources::Cache]
+    # @param http_client [Object] Must implement get method
+    # @param parser [Object] Must implement parse method
+    # @param logger [Logger]
+    # @return [Integer] 
+    def run
+    
+      # Save beginning time
+      start_time = Time.now
+      
+      @logger.info "Running Tango v.#{Tango::VERSION} ..."
+      @logger.info "Target: #{@link_stack.host}."
+      
+      # Use next unlocked database
+      Multidb.use( @db_locker.unlocked )
+      @logger.info "Using database '#{@db_locker.unlocked}'."
+      
+      # Run before filter
+      @logger.info "Loading cache ..."
+      load_cache
+      
+      # Run before filter
+      @logger.info "Running before callback ..."
+      before
+      
+      # Init counter of crawled links
+      links_counter = 0
+      @logger.info "Tango starts crawling ..."
+      
+      # Start crawling website
+      while( @link_stack.has_links? )
+      
+        # Get a link from the stack
+        link = @link_stack.shift 
+        
+        # Skip iteration if no handler found
+        if ! handler_klass = @dispatcher.find_handler( link )
+          @logger.error "No handler for link: #{link}."
+          next
+        end
+        
+        # Try to get contents of the link
+        begin 
+          response = @http_client.get( @link_stack.host + link )
+        rescue StandardError => e 
+          @logger.error "Could not download contents of #{@link_stack.host + link} link."; @logger.error e.message
+          next
+        end
+        
+        # Continue only when response has code 200 or 201
+        if ! [ 200, 201 ].include?( response.code )
+          @logger.error "Response code for link #{link} is #{response.code}. Only code 200 is accepted."
+          next
+        end
+        
+        # Use Nokogiri to parse response contents
+        document = @parser.parse( response.body )
+        
+        # Init handler
+        handler = handler_klass.new( link, document, @cache )
+        
+        # Append links fetched from hanlder
+        @link_stack.append( handler.links )
+        
+        # Try to fire the handler
+        begin
+          handler.trigger
+        rescue StandardError => e  
+          # Log error
+          @logger.error "Link: #{link}. Handler had some troubles."
+          @logger.error e.message
+          @logger.error e.backtrace.join( "\n" )
+        else
+          links_counter += 1
+          @logger.debug "Link: #{link}. Handler triggered successfully."
+        end
+        
+        # Sleep to give crawled server time to breath
+        sleep( @config["sleep"] || 0 )
+        
+      end
+      
+      # Release buffers
+      @logger.info "Releasing buffers ..."
+      @cache.buffer.release_all()
+      
+      # Run after filter
+      @logger.info "Running after callback ..."
+      after
+      
+      # Lock database used in this Tango iteration
+      @db_locker.lock( @db_locker.unlocked )
+    
+      # Get time of script execution ending
+      end_time = Time.now
+      
+      @logger.info "Tango crawled #{links_counter}/#{@link_stack.shifted} links successfully."
+      @logger.info "Start time: #{start_time}, end time: #{end_time}, time elapsed: #{end_time - start_time} seconds."
+      
+      # Close logger
+      @logger.close
+            
+    end
+    
+    private
+    
+    def load_cache
+      
+      @models.each do |symbol, model|
+        model.all.each do |m|
+          @cache.set( symbol, m )
+        end if model.persistent?    
+      end
+      
+    end
+    
+  end
+  
+end

data/lib/tango/database_locker.rb

@@ -0,0 +1,49 @@
+module Tango
+  
+  class DatabaseLocker
+    
+    attr_reader :lock_path
+    
+    #
+    #
+    def initialize( candidates = [], lock_path = "./tmp/database.lock" )
+      @candidates = candidates
+      @lock_path = lock_path
+    end
+    
+    # Return next unlocked database 
+    def unlocked
+      @unlocked ||= find_unlocked
+    end
+    
+    def lock( database )
+      
+      @unlocked = nil
+      
+      File.open( lock_path, "w" ) do |f|
+        f.write( database )
+      end
+      
+      self
+      
+    end
+    
+    private
+    
+    def find_unlocked
+      
+      lock = File.open( lock_path, 'a+' ) { |f| f.read.strip.gsub(/\s+/, ' ') }
+      
+      # If some database was locked use next one
+      unless lock.empty? or ! @candidates.include?( lock )
+        @candidates.at( @candidates.index( lock ).next % @candidates.length )
+      # Otherwise return first one
+      else
+        @candidates.first
+      end
+      
+    end
+    
+  end
+  
+end

data/lib/tango/etl/dispatcher.rb

@@ -0,0 +1,49 @@
+module Tango
+  module ETL
+    
+    # Dispatcher of handlers
+    #
+    # @author Mckomo
+    class Dispatcher
+      
+      def initialize
+        @handlers = []
+      end
+      
+      # Register new handler
+      #
+      # @param  handler_class [HandlerInterface] Class that implements HandlerInterface
+      # @return [Dispatcher]
+      def register( handler_class )
+        
+        # handler must implement HandlerInterface 
+        unless handler_class.ancestors.include? Tango::ETL::HandlerInterface
+          raise "Handler must implement HandlerInterface"
+        end
+        
+        # Append handler to container
+        @handlers << handler_class 
+        
+        self # Chainabilty!
+        
+      end
+      
+      # Find first applicable handler
+      #
+      # @param  url [String]            URL of the page to be handled
+      # @return     [HandlerInterface]
+      def find_handler( url )
+        
+        # Iterate handlers to find first matching handler
+        @handlers.each do |h|
+          return h if h.applicable?( url )
+        end
+        
+        nil
+        
+      end
+      
+    end
+    
+  end
+end

data/lib/tango/etl/handler_interface.rb

@@ -0,0 +1,40 @@
+module Tango
+  
+  module ETL
+    
+    # Handler interface
+    # 
+    # @author Mckomo
+    class HandlerInterface
+      
+      # Constructor of Tango's handler 
+      #
+      #
+      def initialize( link, document, cache = nil )
+        @link = link
+        @document = document
+        @cache = cache      
+      end
+      
+      # 
+      # 
+      # @return [Array|String]
+      def links
+        raise NotImplementedError
+      end
+      
+      # 
+      # 
+      # @return [NilClass]
+      def trigger
+        raise NotImplementedError 
+      end
+      
+      def self.applicable?( link )
+        raise NotImplementedError
+      end
+      
+    end
+    
+  end
+end

data/lib/tango/etl/operator_interface.rb

@@ -0,0 +1,36 @@
+module Tango
+  
+  module ETL
+    
+    # Interface of an operator that implements ETL pattern
+    # 
+    # @author Mckomo
+    class OperatorInterface
+            
+      # Extract resource params
+      #
+      # @param  element [Object]  Element from witch resources should be extracted
+      # @return         [Object]  Extracted resource or array with resources
+      def self.extract( element )
+        raise NotImplementedError
+      end
+      
+      # Transform resource params to desired state
+      # 
+      # @param  resource  [Object]  Resource or array with resources
+      # @return           [Object]  Transformed resource or array with resources
+      def self.transform( resource )
+        raise NotImplementedError
+      end
+      
+      # Load resources into a storage
+      # 
+      # @param  resources   [Array]   Batch of resources to load
+      def self.load( resources )
+        raise NotImplementedError
+      end
+      
+    end
+    
+  end
+end

data/lib/tango/etl.rb

@@ -0,0 +1,3 @@
+require 'tango/etl/dispatcher'
+require 'tango/etl/handler_interface'
+require 'tango/etl/operator_interface'

data/lib/tango/kernel.rb

@@ -0,0 +1,36 @@
+module Tango
+  module Kernel
+    
+    # Convert file path to class name
+    # @param  file_path [String]
+    # @return           [String]
+    def self.classify( file_path )
+      File.basename( file_path, ".*" ).split( "_" ).map { |w| w.capitalize }.join
+    end
+
+    # Load class from a file
+    # 
+    # @param  file          [String]
+    # @param  module_prefix [String]
+    # @return               [Class]
+    def self.load( file, module_prefix = "" )  
+        
+        require file
+        
+        class_name = Kernel.classify( file )
+        Kernel.const_get( "#{module_prefix}#{class_name}" )
+        
+    end
+    
+    # Obtain symbol of a class
+    # @example 
+    #   Tango::Kernel.symblize( A::B::SuperKlass ) #=> :super_klass
+    # 
+    # @param  klass       [Class]
+    # @return             [Symbol]
+    def self.symbolize( klass )
+      klass.to_s.split( '::' ).last.gsub( /(.)([A-Z])/ ,'\1_\2' ).downcase.to_sym
+    end
+
+  end
+end

data/lib/tango/link_stack.rb

@@ -0,0 +1,61 @@
+# Load system lib
+require 'uri'
+
+module Tango
+  
+  # Stack of links to be crawled
+  #
+  # @author Mckomo
+  class LinkStack
+    
+    attr_reader :host, :links, :shifted
+    
+    def initialize( base_link )
+      
+      if base_link !~ URI::regexp
+        raise ArgumentError, "'#{base_link}' is not valid website URL."
+      end
+      
+      # Parse base link
+      url = URI( base_link )
+      
+      @host = "#{url.scheme}://#{url.host}:#{url.port}" # Extract host from base link
+      @links = []                                       # Container for links (without host part)
+      @shifted = 0                                      # Shifted links counter
+      
+      # Extract path (with query) from base link and append it as initial link
+      path = url.query ? "#{url.path}?#{url.query}" : url.path 
+      append( path )
+      
+    end
+    
+    # Shift link from stack and get referrer content
+    #
+    # @return [String]
+    def shift
+      return unless has_links?
+      @shifted += 1 
+      @links.shift
+    end
+
+    
+    # Append link/s to stack
+    #
+    # @return [Array|String] 
+    def append( links )    
+      if links.is_a? String
+        @links << links
+      elsif links.is_a? Array
+        @links += links 
+      end   
+    end
+    
+    # Check if link stack still has links
+    #
+    # @return [Boolean]
+    def has_links?
+      ! @links.empty?
+    end
+     
+  end
+end

data/lib/tango/multidb.rb

@@ -0,0 +1,11 @@
+module Multidb
+  
+  class Balancer
+    attr_reader :candidates
+  end
+  
+  def self.databases
+    balancer.candidates.keys
+  end
+  
+end

data/lib/tango/resource/buffer.rb

@@ -0,0 +1,74 @@
+module Tango
+  module Resource
+    
+    # Resource buffer
+    #
+    # @author Mckomo
+    class Buffer
+      
+      # Constructor of the Buffer
+      #
+      # @param size [Integer]
+      # @return [Tango::Resources::Buffer]
+      def initialize( size = 500 )
+        
+        # Set max size of the buffer
+        @size = size
+      
+        # Init container for resources buffer
+        @resources = {}
+        # Init container for resource operators classes
+        @callbacks = {}
+      
+      end
+      
+      # Register new type of resource to be buffered
+      #
+      # @param type [Symbol]
+      # @param release_callback [Proc]
+      def register( type, &release_callback )
+        
+        raise ArgumentError, "No release callback given" unless block_given?
+        
+        @resources[type] = []
+        @callbacks[type] = release_callback
+        
+      end
+      
+      # Fill buffer with a resource
+      #
+      # @param type [Symbol]
+      # @param resource [Object]
+      def fill( type, resource )
+        
+        raise ArgumentError, "Trying to fill object with unregistered type" unless @resources.keys.include?( type )
+    
+        # Append resource to the buffer
+        @resources[type] << resource
+        # Release the buffer if buffer size exceeded
+        release( type ) if @resources[type].count >= @size
+        
+      end
+      
+      # Release all registered buffers
+      def release_all
+        @resources.keys.each { |type| release( type ) }
+      end
+      
+      private
+      
+      # Release buffer with given type
+      #
+      # @param type [Symbol]
+      def release( type )
+        # Trigger callback on released resources
+        @callbacks[type].tap do |c|
+          c.call( @resources[type] ) 
+        end
+        @resources[type].clear # Clear resources from the buffer
+      end
+      
+    end
+    
+  end
+end