omni_scrapper 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ce4ead1a8478e6bc48b9588ae79f417a7cac6093
-  data.tar.gz: 73022f26551c76efc579773e9d3670a21249b7ac
+  metadata.gz: 8067759f0e4de39da95f44a037f03d65c31ea05a
+  data.tar.gz: be04bec953f8943e3e719f708d8b593dbbc71150
 SHA512:
-  metadata.gz: 218d997e820e1b77eb2629ebed9c67f4b3bdd077082a2cf0dc347d77a9190ec525ff1e82c6dcda84d0389f9bebcd0cab550a0c2911a4fc02d7748ee178844d3f
-  data.tar.gz: 1edd10a0e2f92247e3e4a033763d5b7cd582b0399795c3c745daca21acdec4d384189400488281661a6b5524af23356642b14e73c3cc73b7909245a7dfb18586
+  metadata.gz: 8995c80582d6bda9ed4a317185970d946aa786371fb03a3ac8e0387dc3041d482f24eddaa7d9af954ff25b9d5604e1d16139e00913b2c532b16c994e78c2cac2
+  data.tar.gz: 036a83f2936f563f34050fb5db311be780848c6212de661f045e19dd1a0e72e90046dd39ccc68f574ef56f996bb31160df47d4a2dc268743b1f6c73f002a2868

data/lib/omni_scrapper.rb

@@ -0,0 +1,11 @@
+require 'mechanize'
+require 'omni_scrapper/normalizers'
+require 'omni_scrapper/result'
+require 'omni_scrapper/file_utils'
+require 'omni_scrapper/page'
+require 'omni_scrapper/schema'
+require 'omni_scrapper/configuration'
+require 'omni_scrapper/scrapper_builder'
+require 'omni_scrapper/scrapper'
+
+OmniScrapper::FileUtils.userspace_files.each { |file| require("./#{file}") }

data/lib/omni_scrapper/configuration.rb

@@ -0,0 +1,53 @@
+require 'omni_scrapper/exceptions/crawler_not_defined_exception'
+
+module OmniScrapper
+  class Configuration
+    attr_reader :fields, :anchors
+
+    SINGLE_OPTS = %i[ do method ]
+
+    def initialize
+      @fields = {}
+      @anchors = {}
+    end
+
+    def field(name, options = {})
+      #validate_crawler_presence!(options)
+      validate_field_options!(options)
+      # TODO: validate if field method is defined
+      @fields[name] = options
+    end
+
+    def method_missing(method_name, *args, &block)
+      if args.empty?
+        get_variable(method_name)
+      else
+        set_variable(method_name, args)
+      end
+    end
+
+    private
+
+    def set_variable(name, options)
+      @anchors[name] = { pattern: options[0] }
+    end
+
+    def get_variable(name)
+      # TODO: raise error if unexisting field is requested
+      @anchors[name][:pattern] 
+    end
+
+    def validate_crawler_presence!(options)
+      #return if 
+      fail OmniScrapper::CrawlerNotDefinedException
+    end
+
+    def validate_field_options!(options)
+      incorrect_options = (SINGLE_OPTS & options.keys).first
+      return unless incorrect_options && options.keys.size > 1
+      incompatible_options = (options.keys - [incorrect_options]).map { |i| ":#{i}" }.join(', ')
+      exception_message = ":#{incorrect_options} can not be used with other args(#{incompatible_options})"
+      fail OmniScrapper::InvalidFieldArgumentsException, exception_message
+    end
+  end
+end

data/lib/omni_scrapper/exceptions/crawler_not_defined_exception.rb

@@ -0,0 +1,3 @@
+module OmniScrapper
+  class CrawlerNotDefinedException < StandardError; end;
+end

data/lib/omni_scrapper/exceptions/invalid_field_arguments_exception.rb

@@ -0,0 +1,7 @@
+module OmniScrapper
+  class InvalidFieldArgumentsException < StandardError
+    def initialize(msg = 'Invalid field arguments')
+      super
+    end
+  end
+end

data/lib/omni_scrapper/exceptions/unknown_framework_exception.rb

@@ -0,0 +1,7 @@
+module OmniScrapper
+  class UnknownFrameworkException < StandardError
+    def initialize(msg = 'Uknown framework. Do not know where to generate directory structure.')
+      super
+    end
+  end
+end

data/lib/omni_scrapper/exceptions/unsupported_framework_exception.rb

@@ -0,0 +1,7 @@
+module OmniScrapper
+  class UnsupportedFrameworkException < StandardError
+    def initialize(msg = 'This framework is not supported yet. Consider manual installation.')
+      super
+    end
+  end
+end

data/lib/omni_scrapper/file_utils.rb

@@ -0,0 +1,87 @@
+module OmniScrapper
+  module FileUtils
+    BASE_NAME = ''
+    DIRS = %w( schemas crawlers scrappers normalizers )
+
+    class << self
+      def install
+        Dir.mkdir(installation_location)
+        DIRS.each do |dir|
+          Dir.mkdir("#{installation_location}/#{dir}")
+        end
+      end
+
+      def generate_scrapper(name)
+        Dir.mkdir("#{installation_location}/scrappers/#{name}")
+        File.open("#{installation_location}/scrappers/#{name}/scrapper.rb", "w+") do |f|
+          f.write(scrapper_template(name))
+        end
+
+        File.open("#{installation_location}/scrappers/#{name}/scrap_methods.rb", "w+") do |f|
+          f.write(scrap_methods_template(name))
+        end
+      end
+
+      def userspace_files
+        Dir.glob(File.join(installation_location, "**", "*.rb"))
+      end
+
+      def installed?
+        Dir.exists?(installation_location)
+      end
+
+      def installation_location
+        if hanami?
+          'apps/scrappers'
+        elsif rails?
+          fail OmniScrapper::UnsupportedFrameworkException, 'Rails is not supported yet.'
+        else
+          fail OmniScrapper::UnknownFrameworkException
+        end
+      end
+
+      def hanami?
+        defined? Hanami
+      end
+
+      def rails?
+        defined? Rails
+      end
+
+      def scrapper_template(name)
+        <<-TEMPLATE
+require_relative '../gallery'
+require_relative '../schema'
+
+# Usage example:
+# Scrappers::#{name}::Scrapper.run { |data| p data }
+module Scrappers
+  module #{name}
+    class Scrapper < ::Scrappers::Gallery
+      include OmniScrapper
+
+      setup do |config|
+        config.schema ::Scrappers::Schema
+        config.crawler ::Scrappers::Gallery
+
+        config.entrypoint ''
+        config.next_page_link ''
+
+        config.field :name,
+          selector: ''
+      end
+    end
+  end
+end
+        TEMPLATE
+      end
+
+      def scrap_methods_template(name)
+        <<-TEMPLATE
+module ScrapMethods
+end
+        TEMPLATE
+      end
+    end
+  end
+end

data/lib/omni_scrapper/normalizers.rb

@@ -0,0 +1,6 @@
+require 'omni_scrapper/normalizers/phone'
+
+module OmniScrapper
+  module Normalizers
+  end
+end

data/lib/omni_scrapper/normalizers/base.rb

@@ -0,0 +1,15 @@
+module OmniScrapper
+  module Normalizers
+    class Base
+      attr_accessor :value
+
+      def initialize(value)
+        self.value = value
+      end
+
+      def normalized
+        fail 'Implement in child class'
+      end
+    end
+  end
+end

data/lib/omni_scrapper/normalizers/phone.rb

@@ -0,0 +1,12 @@
+require_relative 'base'
+
+module OmniScrapper
+  module Normalizers
+    class Phone < Base
+      def normalized
+        # TODO: raise exception if phone not presetn
+        value.scan(/([0-9 \-\(\)]{10,})/).flatten.last.gsub(/[ \-\(\)]/, '')
+      end
+    end
+  end
+end

data/lib/omni_scrapper/page.rb

@@ -0,0 +1,92 @@
+module OmniScrapper
+  class Page
+    attr_accessor :page, :config
+
+    def initialize(page, config)
+      self.page = page
+      self.config = config
+    end
+
+    def data
+      result_data = prepare_data
+      validate_data!(result_data)
+    end
+
+    private
+
+    # TODO: should be moved to scrapper
+    def prepare_data
+      config.fields.reduce({}) do |result, (field_name, field_options)|
+        value = get_field(field_options)
+        result.merge(field_name => value)
+      end.merge(id_within_site: id_within_site)
+    end
+
+    def validate_data!(result_data)
+      # TODO: Fix this is not working
+      # schema_pattern method is not available here
+      # uncomment when move this to scrapper level
+      config.schema.new.validate!(result_data) if config.schema
+      result_data
+    end
+
+    def id_within_site
+      extract(page.uri.to_s, config.anchors[:id_within_site][:pattern])
+    end
+
+    def get_field(options)
+      return __send__(options[:method], page) if options[:method]
+      return options[:do].call(page) if options[:do]
+
+      value = find(options[:selector])
+      value = normalize(value, options[:normalizer])
+      value = extract(value, options[:pattern])
+      value = type_cast(value, options[:type_cast_to])
+      value
+    end
+
+    def find(selector)
+      selector = selector.gsub('/tbody', '')
+      page.xpath(selector).text.strip
+    end
+
+    def extract(value, pattern)
+      return value unless pattern
+      value.scan(pattern).flatten.first
+    end
+
+    def type_cast(value, type_class)
+      return value unless type_class
+
+      case type_class.to_s
+      when 'Integer'
+        value.to_i
+      else
+        value
+      end
+    end
+
+    def normalize(value, normalizer)
+      case normalizer
+      when Class
+        normalizer.new(value).normalized
+      when Symbol
+        normalizer_name = normalizer.to_s.split('_').map { |w| w.capitalize }.join
+        normalizer_class = normalizers_namespace.const_get(normalizer_name)
+        normalizer_class.new(value).normalized
+      when Proc
+        normalizer.call(value)
+      when NilClass
+        value
+      end
+    end
+
+    def normalizers_namespace
+      root_namespace.const_get('Normalizers')
+    end
+
+    def root_namespace
+      @root_namespace ||= Kernel.const_get('OmniScrapper')
+    end
+  end
+end

data/lib/omni_scrapper/rake.rb

@@ -0,0 +1,6 @@
+spec = Gem::Specification.find_by_name 'omni_scrapper'
+Dir["#{spec.gem_dir}/lib/tasks/*.rake"].each do |path|
+  load path
+end
+
+

data/lib/omni_scrapper/result.rb

@@ -0,0 +1,29 @@
+module OmniScrapper
+  class Result
+    attr_accessor :scrapper_name, :data, :timestamp, :checksum
+
+    def initialize(scrapper_name)
+      self.scrapper_name = scrapper_name
+      self.timestamp = Time.now
+    end
+
+    def build(data)
+      self.data = data
+      self.checksum = Signature.new(data).calculate
+    end
+
+    private
+
+    class Signature
+      attr_accessor :data
+
+      def initialize(data)
+        self.data = data
+      end
+
+      def calculate
+        Digest::MD5.hexdigest(data.to_s)
+      end
+    end
+  end
+end

data/lib/omni_scrapper/schema.rb

@@ -0,0 +1,13 @@
+require 'json-schema'
+
+module OmniScrapper
+  class Schema
+    def validate!(data)
+      JSON::Validator.validate!(schema, data)
+    end
+
+    def schema
+      fail 'Implement in child class'
+    end
+  end
+end

data/lib/omni_scrapper/scrapper.rb

@@ -0,0 +1,35 @@
+module OmniScrapper
+  def self.included(base)
+    base.extend(OmniScrapper::ClassMethods)
+  end
+
+  class << self
+    def setup(scrapper_name)
+      config = OmniScrapper::Configuration.new
+      yield(config)
+      OmniScrapper::ScrapperBuilder.new(scrapper_name, config).define_classes
+    end
+
+    def scrappers
+      ObjectSpace.each_object(Class).select { |klass| klass < self }
+    end
+  end
+
+  module ClassMethods
+    def run(&block)
+      new.run(&block)
+    end
+  end
+
+  def initialize(entrypoint_url = nil)
+    self.entrypoint = entrypoint_url || entrypoint_pattern
+    self.agent = Mechanize.new do |a|
+      a.user_agent_alias = 'Mac Safari'
+    end
+  end
+
+  def scrap_page(page)
+    data = scrapper_page_class.new(page, configuration).data
+    OmniScrapper::Result.new(name).tap { |result| result.build(data) }
+  end
+end

data/lib/omni_scrapper/scrapper_builder.rb

@@ -0,0 +1,75 @@
+module OmniScrapper
+  class ScrapperBuilder
+    attr_accessor :scrapper_name, :config
+
+    def initialize(scrapper_name, config)
+      self.scrapper_name = scrapper_name
+      self.config = config
+    end
+
+    def define_classes
+      define_scrapper_class(scrapper_name, config)
+      define_page_class
+    end
+
+    private
+
+    def define_scrapper_class(scrapper_name, config)
+      current_module = scrapper_module
+      klass = Class.new(config.crawler) do
+        include OmniScrapper
+
+        config.anchors.each do |name, options|
+          define_method("#{name}_pattern") do
+            options[:pattern]
+          end
+        end
+
+        define_method :configuration do
+          config
+        end
+
+        define_method :name do
+          scrapper_name
+        end
+
+        define_method :scrapper_page_class do
+          current_module.const_get('Page')
+        end
+      end
+
+      Object.const_set(scrappers_namespace_name, Module.new) unless defined? scrappers_namespace_module
+      scrappers_namespace_module.const_set(scrapper_name.to_s.capitalize, Module.new) unless defined? scrapper_module
+      scrapper_module.const_set(scrapper_class_name, klass)
+    end
+
+    def define_page_class
+      page_class = scrapper_module.const_set('Page', Class.new(OmniScrapper::Page))
+      page_class.__send__(:include, class_methods_module) if scrapper_module.const_defined?('ScrapMethods')
+    end
+
+    def scrappers_namespace_module
+      @scrappers_namespace_module ||= Object.const_get(scrappers_namespace_name)
+    end
+
+    def scrappers_namespace_name
+      :Scrappers
+    end
+
+    def scrapper_class_name
+      :Scrapper
+    end
+
+    def scrapper_module
+      @scrapper_module ||= Object.const_get("#{scrappers_namespace_name}::#{scrapper_name.capitalize}")
+    end
+
+    def class_methods_module
+      @class_methods_module ||= Object.const_get scrapper_module_array.push('ScrapMethods').join('::')
+    end
+
+    def scrapper_module_array
+      [scrappers_namespace_name.to_s, scrapper_name.to_s.capitalize]
+    end
+  end
+end

data/lib/tasks/generate.rake

@@ -0,0 +1,13 @@
+require 'omni_scrapper/file_utils'
+
+namespace :omni_scrapper do
+  desc 'Generate scrapper'
+  task :generate, :name do |t, args|
+    if OmniScrapper::FileUtils.installed?
+      OmniScrapper::FileUtils.generate_scrapper(args[:name])
+      p 'Scrapper is generated'
+    else
+      p 'OmniScrapper is not installed. Please run `rake omni_scrapper:install` before.'
+    end
+  end
+end

data/lib/tasks/install.rake

@@ -0,0 +1,17 @@
+require 'omni_scrapper/file_utils'
+
+namespace :omni_scrapper do
+  desc 'Install'
+  task :install do
+    if defined? Hanami
+      p 'Hanami framework detected.'
+
+      if OmniScrapper::FileUtils.installed?
+        p 'OmniScrapper is already installed'
+      else
+        p 'Installing'
+        OmniScrapper::FileUtils.install
+      end
+    end
+  end
+end

data/omni_scrapper.gemspec

@@ -0,0 +1,13 @@
+Gem::Specification.new do |s|
+  s.name          = "omni_scrapper"
+  s.version       = "1.0.1"
+  s.licenses      = ['MIT']
+  s.summary       = "This is an example!"
+  s.description   = "Much longer explanation of the example!"
+  s.author        = "Stanislav Mekhonoshin"
+  s.email         = "ejabberd@gmail.com"
+  s.require_paths = ["lib"]
+  s.files         = `git ls-files`.split($/)
+
+  s.add_runtime_dependency "mechanize"
+end

data/todo.md

@@ -0,0 +1,3 @@
+* Different enqueue policies(for specific scrappers)
+* Additional optional fields(comments to services)
+* Infer fields types from schema, by default

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: omni_scrapper
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Stanislav Mekhonoshin
@@ -29,7 +29,27 @@ email: ejabberd@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
-files: []
+files:
+- lib/omni_scrapper.rb
+- lib/omni_scrapper/configuration.rb
+- lib/omni_scrapper/exceptions/crawler_not_defined_exception.rb
+- lib/omni_scrapper/exceptions/invalid_field_arguments_exception.rb
+- lib/omni_scrapper/exceptions/unknown_framework_exception.rb
+- lib/omni_scrapper/exceptions/unsupported_framework_exception.rb
+- lib/omni_scrapper/file_utils.rb
+- lib/omni_scrapper/normalizers.rb
+- lib/omni_scrapper/normalizers/base.rb
+- lib/omni_scrapper/normalizers/phone.rb
+- lib/omni_scrapper/page.rb
+- lib/omni_scrapper/rake.rb
+- lib/omni_scrapper/result.rb
+- lib/omni_scrapper/schema.rb
+- lib/omni_scrapper/scrapper.rb
+- lib/omni_scrapper/scrapper_builder.rb
+- lib/tasks/generate.rake
+- lib/tasks/install.rake
+- omni_scrapper.gemspec
+- todo.md
 homepage: 
 licenses:
 - MIT