hawker 0.0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 51136fdddf3a7c5b7f533100dea0deb5167569946943d9719b23f6a19f1353f8
+  data.tar.gz: 8d7e8615ad452f0f1145c68762ca5c61cbf387ab95255f945e571322c20198e6
+SHA512:
+  metadata.gz: fc14643a3acc9ff407475b259d078579388f429b1148505aef69ec6daefc06da36fa2fd9dd6696fd6b11ed9df4c52014f8af0b95958533fd3b19d57ddff9668b
+  data.tar.gz: a95638854a9b5b6c9fa4278dcc875c1bedf351b664664f7a27985ca28c30f36b0c704943f1dcfc7cb53d127799f74d0f0e7530a0db680683fcdca7f27f75aa1a

data/lib/hawker.rb

@@ -0,0 +1,29 @@
+require 'hawker/request'
+require 'hawker/drivers/default'
+require 'hawker/drivers/github'
+require 'hawker/drivers/instagram'
+require 'hawker/drivers/twitter'
+require 'hawker/mapper'
+
+module Hawker
+  # Returns the driver instance mapped to the given URL
+  #
+  # @return [Object]
+  def self.get(url)
+    response = Hawker::Request.call(url)
+
+    Hawker::Mapper.map(url, response)
+  end
+
+  # The list of available drivers
+  #
+  # @return [Array<String>]
+  def self.drivers
+    [
+      "GitHub",
+      "Twitter",
+      "Instagram",
+      "Default"
+    ]
+  end
+end

data/lib/hawker/drivers/default.rb

@@ -0,0 +1,32 @@
+require 'nokogiri'
+
+module Hawker
+  module Drivers
+    class Default
+      def initialize(response)
+        @response = response
+      end
+
+      # The current page title
+      #
+      # @return [String]
+      def page_title
+        node_text html.at("head title")
+      end
+
+      private
+
+      attr_reader :response
+
+      def html
+        @html ||= ::Nokogiri::HTML(response.body)
+      end
+
+      def node_text(node)
+        return if node.nil?
+
+        node.text.strip
+      end
+    end
+  end
+end

data/lib/hawker/drivers/github.rb

@@ -0,0 +1,71 @@
+module Hawker
+  module Drivers
+    class Github < Hawker::Drivers::Default
+
+      # The current user full name
+      #
+      # @return [String]
+      def name
+        node_text html.at("h1.vcard-names span.vcard-fullname[itemprop='name']")
+      end
+
+      # The current user nickname
+      #
+      # @return [String]
+      def username
+        node_text html.at("h1.vcard-names span.vcard-username[itemprop='additionalName']")
+      end
+
+      # The current user company
+      #
+      # @return [String]
+      def company
+        node_text html.at("li[itemprop='worksFor'] div")
+      end
+
+      # The current user location
+      #
+      # @return [String]
+      def location
+        node_text html.at("li[itemprop='homeLocation'] span")
+      end
+
+      # The current user website
+      #
+      # @return [String]
+      def website
+        node = html.at("li[itemprop='url'] a")
+        node[:href] unless node.nil?
+      end
+
+      # The count of repositiories owned by the current user
+      #
+      # @return [Integer]
+      def repositories
+        node_text(html.at("a[title='Repositories'] span.Counter")).to_i
+      end
+
+      # The count of stars that the current user gave
+      #
+      # @return [Integer]
+      def stars
+        node_text(html.at("a[title='Stars'] span.Counter")).to_i
+      end
+
+      # The count of users following the current user
+      #
+      # @return [Integer]
+      def followers
+        node_text(html.at("a[title='Followers'] span.Counter")).to_i
+      end
+
+      # The count of users that the current user is following
+      #
+      # @return [Integer]
+      def following
+        node_text(html.at("a[title='Following'] span.Counter")).to_i
+      end
+
+    end
+  end
+end

data/lib/hawker/drivers/instagram.rb

@@ -0,0 +1,66 @@
+require 'json'
+
+module Hawker
+  module Drivers
+    class Instagram < Hawker::Drivers::Default
+
+      # The current user biography
+      #
+      # @return [String]
+      def biography
+        json["entry_data"]["ProfilePage"].first["graphql"]["user"]["biography"]
+      end
+
+      # The current user full name
+      #
+      # @return [String]
+      def full_name
+        json["entry_data"]["ProfilePage"].first["graphql"]["user"]["full_name"]
+      end
+
+      # The current user followers count
+      #
+      # @return [Integer]
+      def followers
+        json["entry_data"]["ProfilePage"].first["graphql"]["user"]["edge_followed_by"]["count"]
+      end
+
+      # The number of accounts that the current user follows
+      #
+      # @return [Integer]
+      def following
+        json["entry_data"]["ProfilePage"].first["graphql"]["user"]["edge_follow"]["count"]
+      end
+
+      # The current user external URL that is present in the biography
+      #
+      # @return [String]
+      def external_url
+        json["entry_data"]["ProfilePage"].first["graphql"]["user"]["external_url"]
+      end
+
+      # The current user profile picture URL
+      #
+      # @return [String]
+      def profile_pic_url
+        json["entry_data"]["ProfilePage"].first["graphql"]["user"]["profile_pic_url"]
+      end
+
+      # The current user Instagram username
+      #
+      # @return [String]
+      def username
+        json["entry_data"]["ProfilePage"].first["graphql"]["user"]["username"]
+      end
+
+      private
+
+      def json
+        @json ||= begin
+          raw_json = html.to_s.match(/window\._sharedData =.*(?=;)/)
+          JSON.parse(raw_json.to_s.gsub("window._sharedData = ", ""))
+        end
+      end
+    end
+  end
+end

data/lib/hawker/drivers/twitter.rb

@@ -0,0 +1,82 @@
+module Hawker
+  module Drivers
+    class Twitter < Hawker::Drivers::Default
+
+      # The current user name
+      #
+      # @return [String]
+      def name
+        node_text(html.at("h1.ProfileHeaderCard-name a"))
+      end
+
+      # The current user biography note
+      #
+      # @return [String]
+      def bio
+        node_text(html.at("p.ProfileHeaderCard-bio"))
+      end
+
+      # The current user location
+      #
+      # @return [String]
+      def location
+        node_text(html.at("span.ProfileHeaderCard-locationText"))
+      end
+
+      # The current user website
+      #
+      # @return [String]
+      def website
+        node = html.at("span.ProfileHeaderCard-urlText a")
+        node[:title] if !node.nil?
+      end
+
+      # The current user account creation date
+      #
+      # @return [String]
+      def joined
+        html.at("span.ProfileHeaderCard-joinDateText")[:title]
+      end
+
+      # The current user tweets count
+      #
+      # @return [Integer]
+      def tweets
+        node = html.at("a[data-nav='tweets'] span.ProfileNav-value")
+        node.nil? ? 0 : node["data-count"].to_i
+      end
+
+      # The current user followers count
+      #
+      # @return [Integer]
+      def followers
+        node = html.at("li.ProfileNav-item--followers span.ProfileNav-value")
+        node.nil? ? 0 : node["data-count"].to_i
+      end
+
+      # The count of accounts that the current user follows
+      #
+      # @return [Integer]
+      def following
+        node = html.at("li.ProfileNav-item--following span.ProfileNav-value")
+        node.nil? ? 0 : node["data-count"].to_i
+      end
+
+      # The current user likes count
+      #
+      # @return [Integer]
+      def likes
+        node = html.at("li.ProfileNav-item.ProfileNav-item--favorites span.ProfileNav-value")
+        node.nil? ? 0 : node["data-count"].to_i
+      end
+
+      # The current user lists count
+      #
+      # @return [Integer]
+      def lists
+        node_text(html.at("li.ProfileNav-item.ProfileNav-item--lists span.ProfileNav-value")).to_i
+      end
+
+    end
+  end
+end

data/lib/hawker/mapper.rb

@@ -0,0 +1,22 @@
+require 'uri'
+
+module Hawker
+  class Mapper
+    MAPPINGS = {
+      /(http|https):\/\/(www.|)github.com\/[a-zA-Z0-9_]*/ => Hawker::Drivers::Github,
+      /(http|https):\/\/(www.|)instagram.com\/[a-zA-Z0-9_]*/ => Hawker::Drivers::Instagram,
+      /(http|https):\/\/(www.|)twitter.com\/[a-zA-Z0-9_]*/ => Hawker::Drivers::Twitter,
+      "default" => Hawker::Drivers::Default
+    }.freeze
+
+    # Maps the given URL to the driver class
+    #
+    # @return [Object]
+    def self.map(url, response)
+      driver_mappings = MAPPINGS.find { |key, value| url.match(key) }
+      driver = driver_mappings.nil? ? MAPPINGS["default"] : driver_mappings.last
+
+      driver.new(response)
+    end
+  end
+end

data/lib/hawker/request.rb

@@ -0,0 +1,13 @@
+require 'rest-client'
+
+module Hawker
+  class Request
+
+    # Performs the request to the given URL in order to get the page html
+    #
+    # @return [RestClient::Response]
+    def self.call(url)
+      ::RestClient.get(url, user_agent: 'Mozilla/5.0 Mac OS')
+    end
+  end
+end

data/lib/hawker/version.rb

@@ -0,0 +1,12 @@
+module Hawker
+  module Version
+    module_function
+
+    # Gem current version
+    #
+    # @return [String]
+    def to_s
+      "0.0.0.1"
+    end
+  end
+end

data/spec/helper.rb

@@ -0,0 +1,5 @@
+def load_driver_fixture(driver)
+  file_path = File.expand_path("../fixtures/#{driver}.html", __FILE__)
+  html_page = File.open(file_path).read
+  response = double('response', body: html_page)
+end

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification
+name: hawker
+version: !ruby/object:Gem::Version
+  version: 0.0.0.1
+platform: ruby
+authors:
+- Paweł Dąbrowski
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-04-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.2
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.2
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.7.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.7.0
+description: Scrap the most popular websites without the knowledge about the page
+  structure or API access
+email: dziamber@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/hawker.rb
+- lib/hawker/drivers/default.rb
+- lib/hawker/drivers/github.rb
+- lib/hawker/drivers/instagram.rb
+- lib/hawker/drivers/twitter.rb
+- lib/hawker/mapper.rb
+- lib/hawker/request.rb
+- lib/hawker/version.rb
+- spec/helper.rb
+homepage: http://github.com/rubyhero/hawker
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.3
+signing_key: 
+specification_version: 4
+summary: Scrap the most popular websites without the knowledge about the page structure
+  or API access
+test_files: []