tango-etl 0.0.1

data/lib/tango/resource/cache.rb

@@ -0,0 +1,81 @@
+module Tango
+  module Resource
+    
+    # Key - value caching system for resources
+    #
+    # @author Mckomo
+    class Cache
+      
+      attr_reader :buffer
+      
+      # Constructor of the cache
+      # 
+      # @param buffer [Tango::Resources::Buffer]
+      # @return [Tango::Resources::Cache]
+      def initialize( buffer = nil )
+        # Set dependencies
+        @buffer = buffer || Buffer.new
+        # Container for resources cache
+        @storage = {}   
+      end
+      
+      # Register new type of resource to be cached
+      #
+      # @param type             [Symbol]
+      # @param release_callback [Proc]
+      def register( type, &release_callback  )
+        # Create container for cache of new resource 
+        @storage[type] = {}
+        # Also register new type with buffer
+        @buffer.register( type, &release_callback )
+      end
+      
+      # Get a resource or use given block to cache and return it's id 
+      #
+      # @param  type      [Symbol]
+      # @param  resource  [Object]
+      # @return           [Integer]
+      def load( type, resource )
+        
+        # Get resource from cache
+        cached_resource = get( type, resource )
+
+        unless cached_resource
+          
+          raise ArgumentError, "No resource callback given" unless block_given?
+          # If not found, execute yield to receive transformed resource
+          cached_resource = yield( resource )
+          # Cache new resource
+          set( type, cached_resource )
+          # Fill buffer with newly cached resource
+          @buffer.fill( type, cached_resource )
+          
+        end
+        
+        cached_resource
+        
+      end
+            
+      # Setter for the cache storage
+      #
+      # @param type [Symbol]
+      # @param resource [Object]
+      # @return [Object]
+      def set( type, resource )
+        raise ArgumentError, "Trying to set resource with unregistered type" unless @storage.keys.include?( type )
+        @storage[type][resource.cache_key] = resource
+      end
+      
+      # Getter for the cache storage
+      #
+      # @param type [Symbol]
+      # @param key [String]
+      # @return [Object]
+      def get( type, resource )
+        raise ArgumentError, "Trying to get resource with unregistered type" unless @storage.keys.include?( type )
+        @storage[type][resource.cache_key]
+      end
+      
+    end
+  end
+end

data/lib/tango/resource.rb

@@ -0,0 +1,2 @@
+require 'tango/resource/buffer'
+require 'tango/resource/cache'

data/lib/tango/version.rb

@@ -0,0 +1,3 @@
+module Tango  
+  VERSION = "0.0.1"
+end

data/lib/tango.rb

@@ -0,0 +1,18 @@
+# Tango == ETl => true
+require 'nokogiri'
+require 'httparty'
+require 'active_record'
+require 'activerecord-import'
+require 'ar-multidb'
+
+require 'yaml'
+require 'logger'
+
+require 'tango/app'
+require 'tango/kernel'
+require 'tango/multidb'
+require 'tango/abstract_model'
+require 'tango/link_stack'
+require 'tango/database_locker'
+require 'tango/etl'
+require 'tango/resource'

data/readme.md

@@ -0,0 +1,3 @@
+# Tango
+
+In short: **Tango is ETL framework**. It's job is to scrap content from target website, `extract` necessary data, `transform` it to desire format and `load` it to a database. 

data/tango.gemspec

@@ -0,0 +1,20 @@
+require './lib/tango/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'tango-etl'
+  s.version     = Tango::VERSION
+  s.license     = 'MIT'
+  s.summary     = 'ETL framework'
+  s.description = 'Scrapping web content made even easier.'
+  s.authors     = ['Maciej Komorowski']
+  s.email       = 'mckomo@gmail.com'
+  s.files       = `git ls-files`.split("\n") - %w[.gitignore]
+  s.test_files  = s.files.select { |p| p =~ /^test\/*test_.*.rb/ }
+  s.homepage    = 'https://github.com/mckomo/Tango'
+  s.add_dependency 'nokogiri', '~> 1.6', '>= 1.6.1'
+  s.add_dependency 'httparty', '~> 0.13', '>= 0.13.1'
+  s.add_dependency 'activerecord', '~> 4.1', '>= 4.1.0'
+  s.add_dependency 'activerecord-import', '~> 0.5', '>= 0.5.0'
+  s.add_dependency 'ar-multidb', '~> 0.1', '>= 0.1.12'
+end
+

data/test/support/db/schema.rb

@@ -0,0 +1,6 @@
+ActiveRecord::Schema.define do
+  create_table :users do |t|
+    t.string      :name
+    t.integer     :age
+  end
+end

data/test/support/lib/model/user.rb

@@ -0,0 +1,11 @@
+module Model
+  
+  class User < Tango::AbstractModel
+  
+    def cache_key
+      [name, age]
+    end
+  
+  end
+  
+end

data/test/support/lib/simple_buffer.rb

@@ -0,0 +1,18 @@
+class SimpleBuffer
+  
+  attr_reader :register_counter, :fill_couter
+  
+  def initialize
+    @register_counter = 0
+    @fill_couter = 0
+  end
+  
+  def register( type )
+    @register_counter += 1
+  end
+  
+  def fill( type, resource )
+    @fill_couter += 1
+  end
+  
+end

data/test/support/lib/simple_handler.rb

@@ -0,0 +1,18 @@
+class SimpleBuffer
+  
+  attr_reader :register_counter, :fill_couter
+  
+  def initialize
+    @register_counter = 0
+    @fill_couter = 0
+  end
+  
+  def register( type )
+    @register_counter += 1
+  end
+  
+  def fill( type, resource )
+    @fill_couter += 1
+  end
+  
+end

data/test/unit/etl/test_dispatcher.rb

@@ -0,0 +1,22 @@
+require "test/unit"
+require 'shoulda'
+require 'mocha/setup'
+
+require 'tango/resource/cache'
+require_relative '../../support/lib/simple_handler.rb'
+
+class TestDispatcher < Test::Unit::TestCase
+      
+  context "a handler dispatcher" do
+      
+    setup do
+      @dispatcher
+    end
+    
+    should "throw exception when trying to load unregistered type" do
+
+    end
+  
+  end
+  
+end

data/test/unit/resource/test_buffer.rb

@@ -0,0 +1,51 @@
+require "test/unit"
+require 'shoulda'
+require 'mocha/setup'
+
+require './lib/tango/resource/buffer.rb'
+
+class TestBuffer < Test::Unit::TestCase
+      
+  context "a buffer" do
+      
+    setup do
+      @buffer_size = 10
+      @buffer = Tango::Resource::Buffer.new( @buffer_size )
+    end
+    
+    should "throw exception when try register type without callback object with unregistered type" do
+      assert_raise ArgumentError do
+        @buffer.register( :foo )
+      end
+    end
+    
+    should "throw exception when try to fill object with unregistered type" do
+      assert_raise ArgumentError do
+        @buffer.fill( :foo, Object.new )
+      end
+    end
+    
+    should "give ability to register new type of resource" do
+      @buffer.register( :foo ) do; nil; end
+      @buffer.fill( :foo, Object.new )
+    end
+    
+    should "trigger release callback when buffer size exceeded" do
+      
+      counter = 0
+      
+      @buffer.register( :foo ) do
+        counter += 1
+      end
+      
+      ( @buffer_size * 5 ).times do 
+        @buffer.fill( :foo, Object.new )
+      end # Full buffer 5 times
+      
+      assert_equal 5, counter
+      
+    end
+  
+  end
+  
+end

data/test/unit/resource/test_cache.rb

@@ -0,0 +1,120 @@
+require "test/unit"
+require 'shoulda'
+require 'mocha/setup'
+
+require 'tango/resource/cache'
+require_relative '../../support/lib/simple_buffer.rb'
+
+class TestCache < Test::Unit::TestCase
+      
+  context "a cache" do
+      
+    setup do
+      @buffer = SimpleBuffer.new
+      @cache = Tango::Resource::Cache.new( @buffer )
+      @foo_resource = stub( cache_key: "foo" )
+    end
+    
+    should "throw exception when trying to load unregistered type" do
+      assert_raise ArgumentError do
+        @cache.load( :foo, "bar" )
+      end
+    end
+    
+    should "throw exception when trying to load yet not cached resource without callback" do
+      
+      @cache.register( :foo ) do
+        nil
+      end
+            
+      assert_raise ArgumentError do
+        @cache.load( :foo, @foo_resource )
+      end
+      
+    end
+    
+    should "be able to store recourse" do
+      
+      @cache.register( :foo ) do
+        nil
+      end
+      
+      @cache.set( :foo, @foo_resource )
+      assert_equal @foo_resource, @cache.get( :foo, @foo_resource )
+      
+    end
+    
+    should "be able to load already stored recourse" do
+      
+      @cache.register( :foo ) do
+        nil
+      end
+      
+      @cache.set( :foo, @foo_resource )
+      assert_equal @foo_resource, @cache.load( :foo, @foo_resource )
+      
+    end
+    
+    should "load yet not cached value using callback" do
+      
+      @cache.register( :foo ) do
+        nil
+      end
+      
+      loaded_resource = @cache.load( :foo, @foo_resource ) do
+        @foo_resource
+      end
+      
+      assert_equal @foo_resource, loaded_resource
+      assert_equal @foo_resource, @cache.load( :foo, @foo_resource )
+      
+    end
+    
+    should "also register new types with buffer" do
+    
+      10.times do |i|
+        @cache.register( i ) do
+          nil
+        end
+      end
+      
+      assert_equal 10, @buffer.register_counter
+      
+    end
+    
+    should "fill buffer with loaded resources" do
+      
+      @cache.register( :foo ) do
+        nil
+      end
+      
+      2.times do |i|
+        resouce = stub( cache_key: i )
+        loaded_resource = @cache.load( :foo, resouce ) do
+          resouce
+        end
+      end
+      
+      assert_equal 2, @buffer.fill_couter
+      
+    end
+    
+    should "fill buffer only once with same resource" do
+      
+      @cache.register( :foo ) do
+        nil
+      end
+      
+      2.times do |i|
+        loaded_resource = @cache.load( :foo, @foo_resource ) do
+          @foo_resource
+        end
+      end
+      
+      assert_equal 1, @buffer.fill_couter
+      
+    end
+  
+  end
+  
+end

data/test/unit/test_abstract_model.rb

@@ -0,0 +1,43 @@
+require 'test/unit'
+require 'shoulda'
+require 'mocha/setup'
+require 'active_record'
+require 'activerecord-nulldb-adapter'
+
+require 'tango/abstract_model'
+require_relative '../support/lib/model/user.rb'
+
+class TestAbstractModel < Test::Unit::TestCase
+      
+  context "instance of a model that extends AbstractModel" do
+    
+    setup do
+      
+      ActiveRecord::Base.establish_connection :adapter => :nulldb, 
+                                              :schema  => Dir.pwd + '/test/support/db/schema.rb'
+      @model = Model::User.new
+      @model.name = "Maciej"
+      @model.age = 22
+      
+    end
+    
+    should "have cache key" do
+      assert_equal ["Maciej", 22], @model.cache_key
+    end
+    
+    should "have array list with its properties values" do
+      assert_equal [nil, "Maciej", 22], @model.values
+    end
+    
+  end
+  
+  context "model class that extends AbstractModel" do
+    
+    should "know what is the next available id" do
+      assert_equal 1, Model::User.next_id
+      assert_equal 2, Model::User.next_id
+    end
+    
+  end
+  
+end

data/test/unit/test_database_locker.rb

@@ -0,0 +1,32 @@
+require 'test/unit'
+require 'shoulda'
+require 'mocha/setup'
+
+require 'tango/database_locker'
+
+class TestDatabaseLocker < Test::Unit::TestCase
+      
+  context "a database locker" do
+    
+    setup do
+      @lock_path = './tmp/database_test.lock'
+      @locker = Tango::DatabaseLocker.new( ["master", "slave"], @lock_path )
+    end
+    
+    teardown do
+      File.delete( @lock_path )
+    end
+    
+    should "create lock file" do
+      @locker.lock( "master" )
+      assert File.exists?( @lock_path )
+      assert_equal "master", IO.read( @lock_path )
+    end
+    
+    should "find unlocked database" do
+       @locker.lock( "slave" )
+       assert_equal "master", @locker.unlocked
+    end
+      
+  end
+end

data/test/unit/test_kernel.rb

@@ -0,0 +1,35 @@
+require 'test/unit'
+require 'shoulda'
+require 'mocha/setup'
+
+require 'tango/kernel'
+
+class TestKernel < Test::Unit::TestCase
+      
+  context "a kernel" do
+    
+    should "transform file path to name of a class" do
+      assert_equal "FooBar", Tango::Kernel.classify( "./../lib/foo_bar.rb" )
+    end
+    
+    should "load a class from a file" do
+      klass = Tango::Kernel.load( Dir.pwd + '/test/support/lib/simple_buffer.rb' )
+      assert_equal SimpleBuffer, klass
+    end
+    
+    should "load a class in a module from a file" do
+      klass = Tango::Kernel.load( Dir.pwd + '/test/support/lib/model/user.rb', 'Model::' )
+      assert_equal Model::User, klass
+    end
+    
+    should "obtain symbol from a class" do
+      assert_equal :simple_buffer, Tango::Kernel.symbolize( SimpleBuffer )
+    end
+    
+    should "obtain symbol from a class in a module" do
+      assert_equal :user, Tango::Kernel.symbolize( Model::User )
+    end
+    
+  end
+  
+end

data/test/unit/test_link_stack.rb

@@ -0,0 +1,49 @@
+require 'test/unit'
+require 'shoulda'
+require 'mocha/setup'
+
+require 'tango/link_stack'
+
+class TestLinkStack < Test::Unit::TestCase
+      
+  context "a link stack" do
+    
+    setup do 
+      @stack = Tango::LinkStack.new( 'http://example.com/data?xml' )
+    end
+    
+    should "raise error when initialized with incorrect URL" do
+      assert_raise ArgumentError do
+        Tango::LinkStack.new( 'ImNotA/Link' )
+      end
+    end
+    
+    should "know host of base link" do
+      assert_equal 'http://example.com:80', @stack.host
+    end
+    
+    should "contain initial link path" do
+      assert @stack.has_links?
+      assert_equal ['/data?xml'], @stack.links 
+      assert_equal '/data?xml', @stack.shift
+      assert ! @stack.has_links?
+    end
+    
+    should "store appended links" do
+      
+      @stack.shift # shift initial path
+      
+      @stack.append '/data/bids'
+      @stack.append [ '/data/bids/1', '/data/bids/2' ]
+      assert_equal 3, @stack.links.count
+      
+      assert_equal '/data/bids', @stack.shift 
+      assert_equal '/data/bids/1', @stack.shift
+      assert_equal '/data/bids/2', @stack.shift
+      
+      assert ! @stack.has_links?
+      
+    end
+      
+  end
+end