apify_core 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 39b891973e8c30065d8136d0b61f4fea6c4c8328
+  data.tar.gz: e2ce4293075a8bb218660a0e0546e86e51559b90
+SHA512:
+  metadata.gz: fdd15b69af5f5068a7f2fadc8afd2f746c074c757fb979569a65eeff525cca225472f21814b1869a20c147f5df3935ad50d40ab61f6a14d82009f50009077c9b
+  data.tar.gz: 57899e4ff289ead9ced7b376b345d4fd9f0b4b1f9b0a53221237724e2510c96bf225e6e44e1694317396bcee20caec8773c5e0fe7b044c02ddfb496e02635fb0

data/Gemfile

@@ -0,0 +1,16 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in apify.gemspec
+gemspec
+
+gem 'activesupport', require: 'active_support/all'
+gem 'watir-webdriver', '~> 0.6.11'
+gem 'rest_client', '~> 1.8.2'
+gem 'headless', '~> 1.0.2'
+gem 'parallel', '~> 1.3.3'
+gem 'nokogiri', '~> 1.6.5'
+
+group :development do
+  gem 'awesome_print', '~> 1.2.0'
+  gem 'pry'
+end

data/Gemfile.lock

@@ -0,0 +1,79 @@
+PATH
+  remote: .
+  specs:
+    apify_core (0.0.1)
+      activesupport
+      headless
+      json
+      nokogiri (~> 1.6.5)
+      parallel
+      rest_client
+      watir-webdriver
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.2.0)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    awesome_print (1.2.0)
+    childprocess (0.5.5)
+      ffi (~> 1.0, >= 1.0.11)
+    diff-lcs (1.2.5)
+    ffi (1.9.6)
+    headless (1.0.2)
+    i18n (0.7.0)
+    json (1.8.1)
+    mini_portile (0.6.2)
+    minitest (5.5.0)
+    multi_json (1.10.1)
+    netrc (0.7.9)
+    nokogiri (1.6.5)
+      mini_portile (~> 0.6.0)
+    parallel (1.3.3)
+    rake (10.4.2)
+    rest_client (1.8.2)
+      netrc (~> 0.7.7)
+    rspec (3.0.0)
+      rspec-core (~> 3.0.0)
+      rspec-expectations (~> 3.0.0)
+      rspec-mocks (~> 3.0.0)
+    rspec-core (3.0.4)
+      rspec-support (~> 3.0.0)
+    rspec-expectations (3.0.4)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.0.0)
+    rspec-mocks (3.0.4)
+      rspec-support (~> 3.0.0)
+    rspec-support (3.0.4)
+    rubyzip (1.1.6)
+    selenium-webdriver (2.44.0)
+      childprocess (~> 0.5)
+      multi_json (~> 1.0)
+      rubyzip (~> 1.0)
+      websocket (~> 1.0)
+    thread_safe (0.3.4)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    watir-webdriver (0.6.11)
+      selenium-webdriver (>= 2.18.0)
+    websocket (1.2.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport
+  apify_core!
+  awesome_print (~> 1.2.0)
+  bundler (~> 1.7)
+  headless (~> 1.0.2)
+  nokogiri (~> 1.6.5)
+  parallel (~> 1.3.3)
+  rake (~> 10.0)
+  rest_client (~> 1.8.2)
+  rspec (~> 3.0.0)
+  watir-webdriver (~> 0.6.11)

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 victorvsk
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,43 @@
+# Apify Core
+
+Apify Core is a part of Apify Project. Parse HTML\XML to JSON with easy API and useful filters.
+Apify Project allows even more - parsing entire website with east.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'apify_core'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install apify_core
+
+## Usage
+
+```
+html = RestClient.get('http://github.com')
+pattern = { title: '<% title %>' }
+title = Apify::Core.new(html, pattern).perform # GitHub · Build software better, together.
+```
+
+```
+request = { github: { url: ['http://github.com'], js: false, host: 'http://github.com', pattern: { title: '<% title %>' } } }
+response = Apify.crawl!(request) # { "github": { "title": "GitHub · Build software better, together." } }
+```
+
+See more in documentation (TODO). Also some syntax examples can be found in spec/examples.
+
+## Contributing
+
+1. Fork it ( https://github.com/victorvsk/apify-core/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,3 @@
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('./Gemfile', __FILE__)
+require 'rubygems'
+require "bundler/gem_tasks"

data/apify_core.gemspec

@@ -0,0 +1,34 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'apify_core/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "apify_core"
+  spec.version       = Apify::Core::VERSION
+  spec.authors       = ["victorvsk"]
+  spec.email         = ["victor@vyskrebentsev.ru"]
+  spec.summary       = %q{Core part of Apify project. An easy way to parse HTML\XML content and crawl websites in a normalized and centralized way.}
+  spec.description   = %q{Simple API to transform from simple HTML to JSON to entire website to JSON.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0.0"
+
+
+  spec.add_dependency 'watir-webdriver'
+  spec.add_dependency 'rest_client'
+  spec.add_dependency 'headless'
+  spec.add_dependency 'parallel'
+  spec.add_dependency 'nokogiri', '~> 1.6.5'
+  spec.add_dependency 'json'
+  spec.add_dependency 'activesupport'
+
+end

data/bin/bundler

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'bundler' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('bundler', 'bundler')

data/bin/coderay

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'coderay' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('coderay', 'coderay')

data/bin/htmldiff

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'htmldiff' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('diff-lcs', 'htmldiff')

data/bin/ldiff

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'ldiff' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('diff-lcs', 'ldiff')

data/bin/nokogiri

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'nokogiri' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('nokogiri', 'nokogiri')

data/bin/pry

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'pry' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('pry', 'pry')

data/bin/rackup

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'rackup' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('rack', 'rackup')

data/bin/rake

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'rake' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('rake', 'rake')

data/bin/rspec

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'rspec' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('rspec-core', 'rspec')

data/bin/server

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'server' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('apify', 'server')

data/bin/tilt

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'tilt' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('tilt', 'tilt')

data/lib/apify_core/fetcher.rb

@@ -0,0 +1,190 @@
+module Apify
+  module Core
+    class Fetcher
+      attr_accessor :sources
+      attr_accessor :result
+
+      def initialize( pages, processes=2, delay=0 )
+        @pages = pages
+        @processes = processes
+        @delay = delay
+      end
+
+      def prepare
+        @pages.each do |key, value|
+          value[:url] = Filter.apply(value[:url], ['map_urlencode'])
+          self.class.send(:attr_reader, key.to_sym)
+          js  = value[:js] || false
+          pattern = value[:pattern]
+          host = value[:host]
+          if value[:from]
+            v =  value
+            instance_variable_set("@#{key}".to_sym, v)
+            next
+          end
+          url = self.class.base_url_for(value[:url], value[:host])
+
+          if value[:paginate]
+            pages = self.class.paginate(url: url, to_replace: value[:paginate][0], pagination: value[:paginate][1])
+            result = { pages: pages }
+          else
+            url = (url.respond_to?(:each) ? url : url.to_s)
+            result = { pages: [url] }
+          end
+          result[:js] = js
+          result[:pattern] = pattern
+          result[:host] = host
+          instance_variable_set("@#{key}".to_sym, result)
+
+        end
+        self.sources = @pages.keys
+      end
+
+      def perform
+        parenthesis_args = /\([\"\'](.*?)[\"\']\)/
+        self.sources.each do |source|
+
+          if @pages[source].key?(:from)
+            expression = @pages[source][:from]
+            statement = {
+              select:   expression.match(/select#{parenthesis_args}/)[1],
+              from:     expression.match(/from#{parenthesis_args}/)[1],
+              filters:  (expression.match(/filter#{parenthesis_args}/)[1].split('|').map(&:strip) rescue nil),
+            }
+            statement[:filters] = ['mapattr_href', 'map_urlencode'] unless statement[:filters].present?
+            urls = Filter.apply(Parser.fetch(statement[:select], self.send(statement[:from])[:pages]), statement[:filters])
+            @pages[source][:pages] = self.class.base_url_for(urls, @pages[source][:host])
+          end
+          src = self.send(source)
+          method = src[:js] ? :js : :normal
+          processes_number = (method == :js ? 1 : @processes )
+          src[:pages] = ::Parallel.map(src[:pages], in_processes: processes_number) do |url_or_array|
+            if url_or_array.respond_to?(:each)
+              res = []
+              url_or_array.each do |url|
+
+                res << self.class.download( url, method )
+              end
+              res
+            else
+              self.class.download( url_or_array, method )
+            end
+          end.flatten
+        end
+
+        result = {}
+        self.sources.each do |source|
+          src = self.send(source)
+          pattern = src[:pattern] ? src[:pattern].dup : false
+          elem = if src[:pattern]
+            src[:pages].map{ |html| Parser.new(html, src[:pattern]).perform }
+          else
+
+            src[:pages]
+          end
+
+          result[source] = elem if pattern
+          instance_variable_set("@#{source}".to_sym, elem )
+        end
+
+        @result = result
+        @json = result
+      end
+
+      def to_json
+        (@json || perform).to_json
+      end
+
+      class << self
+
+        # Fetcher::Fetcher.paginate(url: 'http://site.com', to_replace: '(\/?)$', pagination: '?page=<% 1,5,1 %>')
+
+        def paginate( opts={} )
+          pagination = opts[:pagination] || '?page=<% 1,5,1 %>'
+          to_replace = opts[:to_replace] || '(\/?)\Z'
+          url_or_array_of_urls = opts[:url]
+          raise ArgumentError, "URL parameter missing" if url_or_array_of_urls.nil?
+          regexp = /<%\s?+(\d+,\d+,\d+)\s?+%>/
+          pattern = pagination.scan(regexp)
+          return [opts[:url]] if pattern.count == 0
+          raise ArgumentError, "Only one pagination pattern allowed." if pattern.count > 1
+          result = []
+          pager_args = pattern.first.first.split(',').map(&:strip).map(&:to_i)
+          range = (pager_args[0]..pager_args[1])
+          range.step(pager_args[2]).each do |page|
+
+            to_append = pagination.gsub(regexp, page.to_s)
+            if url_or_array_of_urls.respond_to?(:each)
+              url_or_array_of_urls.each do |url|
+                result << url.to_s.chomp('/').gsub(Regexp.new(to_replace), to_append)
+              end
+            else
+              result << url_or_array_of_urls.to_s.chomp('/').gsub(Regexp.new(to_replace), to_append)
+            end
+
+          end
+
+          result
+        end
+
+        def download( url, method=:normal )
+          sleep @delay if @delay.to_i > 0
+          result = case method
+          when :js
+            headless = Headless.new
+            headless.start
+            browser = Watir::Browser.new
+            browser.goto url
+            html = browser.html
+            headless.destroy
+            print "+"
+            html
+          when :normal
+            begin
+              html = RestClient.get(url,
+                      'Accept'          => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+                      'Accept-Language' => 'ru-RU,ru;q=0.8,en-US;q=0.6,en;q=0.4',
+                      'Connection'      => 'keep-alive',
+                      'User-Agent'      => 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/39.0.2171.65 Chrome/39.0.2171.65 Safari/537.36')
+              d = Nokogiri::HTML(html)
+              charset = d.search('meta[@http-equiv="content-type"]').last['content'].match(/charset=(.+)/)[1].downcase rescue nil
+              if charset.present? and charset != 'utf-8'
+                html = html.force_encoding(charset).encode("utf-8", undef: :replace)
+              end
+              print "+"
+              html
+            rescue RestClient::RequestTimeout, RestClient::ResourceNotFound, RestClient::InternalServerError, URI::InvalidURIError, RestClient::Forbidden,RestClient::BadGateway, RestClient
+              print "-"
+              return
+            end
+          end
+
+          result
+        end
+
+        def base_url_for(url_or_array, base_url)
+
+          if url_or_array.respond_to?(:each)
+            result = []
+            url_or_array.each do |url|
+              #url = URI(URI.encode(url))
+              url = URI(url)
+              raise ArgumentError, "No host provided." if url.host.nil? and base_url.nil?
+              result << (url.host ? url.to_s : "#{base_url}#{url}")
+            end
+            result
+          else
+            #url = URI(URI.encode(url_or_array))
+            url = URI(url_or_array)
+            raise ArgumentError, "No host provided." if url.host.nil? and base_url.nil?
+            (url.host ? url : "#{base_url}#{url}")
+          end
+
+        end
+
+      end
+
+
+    end
+  end
+end

data/lib/apify_core/filter.rb

@@ -0,0 +1,83 @@
+module Apify
+  module Core
+    module Filter
+      class << self
+
+        def apply(node_or_str, filters=[])
+          return node_or_str if filters.empty? or node_or_str.nil?
+          method = filters.shift
+          filtered_value = send(method, node_or_str)
+          apply(filtered_value, filters)
+        end
+
+        private
+
+        def first(node)
+          node.first
+        end
+
+        def text(node)
+          node.text
+        end
+
+        def strip(str)
+          str.strip if str
+        end
+
+        def list(node)
+          node
+        end
+
+        def html(node)
+          node.to_s
+        end
+
+        def inner_html(node)
+          node.inner_html.to_s
+        end
+
+        def map_text(node)
+          node.map(&:text).map(&:strip)
+        end
+
+        def map_html(node)
+          node.map(&:to_s)
+        end
+
+        def map_inner_html(node)
+          node.map(&:inner_html).map(&:to_s)
+        end
+
+
+        def map_urlencode(node)
+          node.map do |url|
+            urlencode(url)
+          end
+        end
+
+        def urlencode(url)
+          url = begin
+            url = URI(url)
+            url
+          rescue URI::InvalidURIError
+            URI.encode(url)
+          end
+        end
+
+        def method_missing(method_sym, *arguments, &block)
+          if method_sym =~ /\Amapattr_/
+            attribute = method_sym.to_s.gsub('mapattr_', '')
+            arguments.first.map{ |n| n[attribute] }
+          elsif method_sym =~ /\Aattr_/
+            attribute = method_sym.to_s.gsub('attr_', '')
+            arguments.first[attribute]
+          else
+            super
+          end
+        end
+
+
+      end
+    end
+  end
+end