skyscraper 0.0.1

data/lib/skyscraper/node/resource.rb

@@ -0,0 +1,57 @@
+module Skyscraper
+  module Node
+    class Resource
+      def initialize node
+        @node           = node
+        @path           = extract_path_from_node(@node)
+      end
+
+      def download options = {}
+        @name          = options[:file_name] || @path.file_name
+        @new_file_path = replace_path_variables(options[:path] || Skyscraper.config.download_path)
+        @temp_file     = open(@path.full_path)
+
+        copy @temp_file.path, @new_file_path
+        @new_file_path
+      end
+
+      private 
+
+      def copy from, to 
+        create_path_if_not_exists to
+        `cp #{from} #{to}`
+      end
+
+      def create_path_if_not_exists path
+        `mkdir -p #{path}` unless File.directory?(path)
+      end
+
+      def replace_path_variables path
+        new_path = path.dup
+        new_path.gsub! /:file_name/, @name
+        new_path.gsub! /:sequence/, get_sequence_number_for(new_path)
+        new_path
+      end
+
+      def get_sequence_number_for path
+        new_path = path.split(":sequence")[0]
+        if File.directory?(new_path)
+          entries = Dir.entries(new_path).select { |i| i =~ /^\d+$/ } || []
+          last = entries.sort.last.to_i
+          last += 1
+          last.to_s
+        else
+          "1"
+        end
+      end
+
+      def extract_path_from_node node
+        if href_or_src = node.href.present? ? node.href : node.src
+          node.element.document.path.path_for(href_or_src)
+        else
+          throw Exception.new("no href no src")
+        end
+      end
+    end
+  end
+end

data/lib/skyscraper/pages.rb

@@ -0,0 +1,27 @@
+require "open-uri"
+
+module Skyscraper
+  class Pages
+    attr_accessor :items
+
+    def initialize value = nil, &block
+      set value, &block
+    end
+
+    def set value = nil, &block
+      @items = block ? block.call(Skyscraper) : value
+      @items = @items.is_a?(Array) ? @items : [@items]
+      @items.flatten!
+      reset
+      self
+    end
+
+    def next
+      @items[@current += 1] 
+    end
+
+    def reset
+      @current = -1
+    end
+  end
+end

data/lib/skyscraper/path.rb

@@ -0,0 +1,29 @@
+module Skyscraper
+  module Path
+    extend ActiveSupport::Autoload
+
+    autoload :Base
+    autoload :Local
+    autoload :Remote
+
+    class << self
+      def factory path
+        if Path.remote?(path)
+          Path::Remote.new(path)
+        else
+          Path::Local.new(path)
+        end
+      end
+
+      def remote? path
+        path = "http://"+path unless path.match /^(http|https):\/\//
+        uri  = URI.parse(path)
+        uri.host ? true : false
+      end
+
+      def absolute? path
+        path.starts_with? "/"
+      end
+    end
+  end
+end

data/lib/skyscraper/path/base.rb

@@ -0,0 +1,15 @@
+module Skyscraper
+  module Path
+    class Base
+      def path_for path
+        Path::factory(self.full_path_for(path))
+      end
+
+      private
+
+      def get_file_name path
+        path.last == "/"  ? nil : path.split("/").last
+      end
+    end
+  end
+end

data/lib/skyscraper/path/local.rb

@@ -0,0 +1,29 @@
+module Skyscraper
+  module Path
+    class Local < Path::Base
+      attr_accessor :base, :folder, :file_name, :full_path
+
+      def initialize path
+        @folder    = get_folder(path)
+        @full_path = path
+        @file_name = get_file_name(path)
+        @base      = @folder
+      end
+
+      def full_path_for href
+        Path.absolute?(href) ? href : "#{@folder}#{href}"
+      end
+
+      def to_s
+        self.full_path
+      end
+
+      private
+
+      def get_folder path
+        path.match(/\/.+\//)[0]
+      end
+    end
+  end
+end
+

data/lib/skyscraper/path/remote.rb

@@ -0,0 +1,32 @@
+module Skyscraper
+  module Path
+    class Remote < Path::Base
+      attr_accessor :uri, :base, :domain, :full_path, :path, :query, :file_name
+
+      def initialize path
+        uri = URI.parse(path)
+
+        if uri.scheme.present?
+          @domain = uri.scheme + "://" + uri.host
+        else
+          @domain = uri.host
+        end
+
+        @path      = uri.path
+        @query     = "?" + uri.query if uri.query
+        @full_path = "#{@domain}#{@path}#{@query}"
+        @base      = "#{@domain}/"
+        @uri       = uri
+        @file_name = get_file_name(@path)
+      end
+
+      def full_path_for href
+        @uri.merge(URI.parse(href)).to_s
+      end
+
+      def to_s
+        self.full_path
+      end
+    end
+  end
+end

data/lib/skyscraper/results.rb

@@ -0,0 +1,93 @@
+module Skyscraper
+  class Results
+    attr_accessor :limit, :delay, :after_each, :after_all, :records
+
+    def initialize base, options = {}
+      @delay         = extract_delay_hash(options[:delay] || base.config.delay)
+      @limit         = options[:limit] || base.config.limit
+
+      @base          = base
+
+      @after_each    = options[:after_each] || []
+      @after_all     = options[:after_all]  || []
+
+      @records       = []
+    end
+
+    def add_after_each &block
+      @after_each << block
+    end
+
+    def add_after_all &block
+      @after_all << block
+    end
+
+    def fetch continue = false
+      results   = []
+      documents = []
+
+      @base.pages_object.reset unless continue
+
+      i = 0
+
+      while i != @limit and page = @base.pages_object.next
+        result = {}
+
+        begin
+          document = Skyscraper::fetch(page)
+
+          @base.fields.each do |field|
+            result[field.name] = field.find_in_document document 
+          end
+
+          call_callbacks @after_each, result, document
+          results << result
+          sleep @delay[:sleep] if (i+1) % @delay[:after] == 0
+
+        rescue SocketError, Errno::ENOENT
+          warning_msg = "WARNGIN: resource '#{page}' not found!"
+          puts warning_msg if @base.config.noise_errors
+          raise NoResourceException, warning_msg unless @base.config.skip_on_error
+        end
+
+        i += 1
+      end
+      
+      call_callbacks @after_all, results
+
+      @records += results
+      results
+    end
+
+    def continue
+      fetch true
+    end
+
+    private
+
+    def extract_delay_hash delay_hash
+      delay = {}
+
+      if delay_hash and delay_hash.is_a? Hash
+        delay = delay_hash
+      elsif delay_hash
+        delay[:sleep] = delay_hash
+        delay[:after] = 1
+      else
+        delay[:sleep] = 0
+        delay[:after] = 1
+      end
+
+      delay
+    end
+
+    def call_callbacks callbacks, *args
+      callbacks.each do |callback|
+        callback.call(*args)
+      end
+    end
+  end
+
+  class NoResourceException < Exception 
+  end
+end

data/lib/version.rb

@@ -0,0 +1,3 @@
+module Skyscraper
+  VERSION = "0.0.1"
+end

data/skyscraper.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/./version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Adam Dratwinski"]
+  gem.email         = ["arboooz@gmail.com"]
+  gem.description   = %q{Library that helps scraping data from websites in easy way}
+  gem.summary       = %q{Library that helps scraping data from websites in easy way}
+  gem.homepage      = "https://github.com/boooz/skyscraper"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(spec)/})
+  gem.name          = "skyscraper"
+  gem.require_paths = ["lib"]
+  gem.version       = Skyscraper::VERSION
+
+  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "rake"
+  gem.add_dependency "nokogiri"
+  gem.add_dependency "actionpack"
+end

data/spec/skyscraper/skyscraper/base_spec.rb

@@ -0,0 +1,83 @@
+describe Skyscraper::Base do
+
+  it "should set pages" do
+    base = Skyscraper::Base.new
+    base.pages "http://onet.pl"
+    base.pages_object.is_a? Skyscraper::Pages
+  end
+
+  it "should have defaults for config" do
+    Skyscraper.config.encoding = "utf-8"
+    base = Skyscraper::Base.new
+    base.config.encoding.should == "utf-8"
+  end
+
+  it "should be able to have different config for each instance" do
+    base_a = Skyscraper::Base.new 
+    base_a.config.bar = "foo"
+    base_a.settings foo: "biz"
+
+    base_b = Skyscraper::Base.new 
+    base_b.config.bar = "biz"
+    base_b.settings foo: "bar"
+
+    base_a.config.bar.should == "foo"
+    base_a.config.foo.should == "biz"
+    base_b.config.bar.should == "biz"
+    base_b.config.foo.should == "bar"
+  end
+
+  it "should add fields" do
+    base = Skyscraper::Base.new
+    base.field :name, ".selector"
+    base.field :other, ".selector"
+    base.fields.length.should == 2
+  end
+
+  it "should override fields with the same name" do
+    base = Skyscraper::Base.new
+    base.field :name, ".selector"
+    base.field :name, ".selector"
+    base.fields.length.should == 1 
+  end
+
+  it "should add after each callback" do
+    base = Skyscraper::Base.new
+    base.pages path_to("skyscraper-base.html")
+    base.field :h1, "h1"
+    base.after_each { |result, page| result[:h1] += "2" }
+    base.fetch[0][:h1].should == "Hello world2"
+  end
+
+  it "should add after all callback" do
+    base = Skyscraper::Base.new
+    base.pages path_to("skyscraper-base.html")
+    base.field :h1, "h1"
+    base.after_all { |results| results << "2" }
+    base.fetch.length.should == 2
+  end
+
+  it "should set settings" do 
+    base = Skyscraper::Base.new 
+    base.settings limit: 100, delay: { sleep: 2, after: 10 }
+    base.config.limit.should == 100
+    base.config.delay[:sleep].should == 2
+    base.config.delay[:after].should == 10
+  end
+
+  it "should fetch data" do
+    base = Skyscraper::Base.new
+    base.pages path_to("skyscraper-base.html")
+    base.field :h1, "h1"
+    base.fetch[0][:h1].should == "Hello world"
+  end
+
+  it "should be able to continue fetching" do
+    Skyscraper.config.limit = 10
+    base = Skyscraper::Base.new
+    base.pages [path_to("skyscraper-base.html")] * 12
+    base.field :h1, "h1"
+    base.fetch.length.should == 10
+    base.continue.length.should == 2
+  end
+end

data/spec/skyscraper/skyscraper/config_spec.rb

@@ -0,0 +1,25 @@
+describe Skyscraper::Config do
+  it "should set variable on initialize" do
+    config = Skyscraper::Config.new foo: "bar"
+    config.foo.should == "bar"
+  end
+
+ it "should set dynamic variable" do
+    config = Skyscraper::Config.new foo: "bar"
+    config.foo.should == "bar"
+    config.bar = "foo"
+    config.bar.should == "foo"
+  end
+
+  it "should override variable value" do
+    config = Skyscraper::Config.new foo: "bar"
+    config.foo = "bizz"
+    config.foo.should == "bizz"
+  end
+
+  it "should override false value" do
+    config = Skyscraper::Config.new foo: true
+    config.foo = false
+    config.foo.should == false
+  end
+end

data/spec/skyscraper/skyscraper/document_spec.rb

@@ -0,0 +1,14 @@
+#encoding: utf-8
+
+describe Skyscraper::Document do
+  it "should support utf-8 encoding by default in remote pages" do
+    document = Skyscraper::Document::load("http://www.sjp.pl/grzegrz%F3%B3ka")
+    document.encoding.should == "utf-8"
+    document.css(".lc").first.content.strip == "Grzegrzółka"
+  end
+
+  it "should have path" do
+    document = Skyscraper::Document::load(path_to("skyscraper-document.html"))
+    document.path.should be_an Skyscraper::Path::Base
+  end
+end