browze 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 72b282f5f06f2f9689bf7f5cff3b6f048673f423e034933aed3336d39b9605ce
+  data.tar.gz: 43b47d3e8cb2ca20390ed40f349cd8c65c53ebb382e6f56dfb736d5c4e2feffa
+SHA512:
+  metadata.gz: 3d16893d0da113ccc55de31d487bb39f7a8ed9378a457a4046ab692dda56d529f29457f3cf50e0389ed09d8dce376c686b2d8501f3649c5e2de35ae2ff7153ba
+  data.tar.gz: 62fb2a6483075b6199b80b3c6d63b5361690e36d3ddc31503568e50acb64bb6ae067d84b7640caf461af4d4d9f5c0f0b0d8bb64f621495dbce7d949a37fb9255

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.vscode/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+Gemfile.lock

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,6 @@
+---
+language: ruby
+cache: bundler
+rvm:
+  - 2.6.5
+before_install: gem install bundler -v 2.1.4

data/Gemfile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in browze.gemspec
+gemspec
+
+gem 'guard-rspec', require: false
+gem 'rake', '~> 12.0'
+gem 'rspec', '~> 3.0'

data/Guardfile

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+## Uncomment and set this to only include directories you want to watch
+# directories %w(app lib config test spec features) \
+#  .select{|d| Dir.exist?(d) ? d : UI.warning("Directory #{d} does not exist")}
+
+## Note: if you are using the `directories` clause above and you are not
+## watching the project directory ('.'), then you will want to move
+## the Guardfile to a watched dir and symlink it back, e.g.
+#
+#  $ mkdir config
+#  $ mv Guardfile config/
+#  $ ln -s config/Guardfile .
+#
+# and, you'll have to watch "config/Guardfile" instead of "Guardfile"
+
+# Note: The cmd option is now required due to the increasing number of ways
+#       rspec may be run, below are examples of the most common uses.
+#  * bundler: 'bundle exec rspec'
+#  * bundler binstubs: 'bin/rspec'
+#  * spring: 'bin/rspec' (This will use spring if running and you have
+#                          installed the spring binstubs per the docs)
+#  * zeus: 'zeus rspec' (requires the server to be started separately)
+#  * 'just' rspec: 'rspec'
+
+guard :rspec, cmd: 'bundle exec rspec', failed_mode: :focus, all_after_pass: true do
+  require 'guard/rspec/dsl'
+  dsl = Guard::RSpec::Dsl.new(self)
+
+  # Feel free to open issues for suggestions and improvements
+
+  # RSpec files
+  rspec = dsl.rspec
+  watch(rspec.spec_helper) { rspec.spec_dir }
+  watch(rspec.spec_support) { rspec.spec_dir }
+  watch(rspec.spec_files)
+
+  # Ruby files
+  ruby = dsl.ruby
+  dsl.watch_spec_files_for(ruby.lib_files)
+  watch('lib/browze/client.rb') { 'spec/client' }
+end

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Gianpiero Addis
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,69 @@
+# Browze
+
+A scraping-oriented browser-like wrapper around HTTParty for your cli tools.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'browze'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install browze
+
+## Usage Examples
+
+```ruby
+# Instantiate the browser
+browser = Browze.start
+
+# By default the browser is instantiated with a desktop user agent. if you want
+# to use a mobile user agent instead, start it with :mobile.
+mobile_browser = Browze.start(:mobile)
+
+# A random user agent is set automatically
+# e.g.: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"
+browser.user_agent
+
+# Set any custom headers
+browser.headers = { foo: 'bar' }
+
+# Perform a GET request
+response = browser.get('https://www.google.com/')
+
+# Return the parsed response body as a Nokogiri::HTML::Document
+response.parsed
+
+# Perform a POST request
+browser.post(
+  'https://www.example.com/url.php',
+  param1: 'example',
+  param2: 'example'
+)
+
+# Download a file and show a progress bar
+browser.download('https://www.example.com/robots.txt')
+
+# Show the current IP and geolocation
+browser.ip       # => 75.152.126.127
+browser.location # => Grande Prairie, Canada
+```
+
+## Development
+
+After checking out the repo, run `bundle install` to install dependencies. Then, run `bundle exec rspec` to run the specs or `bundle exec guard --clear` to keep them running during the development.
+
+You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "browze"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/browze.gemspec

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative 'lib/browze/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'browze'
+  spec.version       = Browze::VERSION
+  spec.authors       = ['Gianpiero Addis']
+
+  spec.summary       = 'A wrapper around HTTParty to quick start scraping tasks.'
+  spec.license       = 'MIT'
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.3.0')
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'down'            # https://github.com/janko/down
+  spec.add_dependency 'geocoder'        # https://github.com/alexreisner/geocoder
+  spec.add_dependency 'httparty'        # https://github.com/jnunemaker/httparty
+  spec.add_dependency 'nokogiri'        # https://github.com/sparklemotion/nokogiri
+  spec.add_dependency 'tty-progressbar' # https://github.com/piotrmurach/tty-progressbar
+
+  spec.add_development_dependency 'rspec-retry'
+  spec.add_development_dependency 'vcr'
+  spec.add_development_dependency 'webmock'
+end

data/lib/browze/client/desktop.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Browze
+  class Client::Desktop < Browze::Client
+    # @return [Array] Most used desktop user agents.
+    USER_AGENTS = [
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36',
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36',
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Safari/605.1.15',
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36',
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0',
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36',
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36',
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:75.0) Gecko/20100101 Firefox/75.0',
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36',
+      'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:76.0) Gecko/20100101 Firefox/76.0',
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:76.0) Gecko/20100101 Firefox/76.0',
+      'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36',
+      'Mozilla/5.0 (Windows NT 10.0; rv:68.0) Gecko/20100101 Firefox/68.0',
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
+    ].freeze
+  end
+end

data/lib/browze/client/mobile.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Browze
+  class Client::Mobile < Browze::Client
+    # @return [Array] Most used mobile user agents.
+    USER_AGENTS = [
+      'Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1',
+      'Mozilla/5.0 (Linux; U; Android 4.4.2; en-us; SCH-I535 Build/KOT49H) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30',
+      'Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36',
+      'Mozilla/5.0 (Linux; Android 7.0; SM-A310F Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.91 Mobile Safari/537.36 OPR/42.7.2246.114996',
+      'Opera/9.80 (Android 4.1.2; Linux; Opera Mobi/ADR-1305251841) Presto/2.11.355 Version/12.10',
+      'Opera/9.80 (J2ME/MIDP; Opera Mini/5.1.21214/28.2725; U; ru) Presto/2.8.119 Version/11.10',
+      'Mozilla/5.0 (iPhone; CPU iPhone OS 7_1_2 like Mac OS X) AppleWebKit/537.51.2 (KHTML, like Gecko) OPiOS/10.2.0.93022 Mobile/11D257 Safari/9537.53',
+      'Mozilla/5.0 (Android 7.0; Mobile; rv:54.0) Gecko/54.0 Firefox/54.0',
+      'Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_2 like Mac OS X) AppleWebKit/603.2.4 (KHTML, like Gecko) FxiOS/7.5b3349 Mobile/14F89 Safari/603.2.4',
+      'Mozilla/5.0 (Linux; U; Android 7.0; en-US; SM-G935F Build/NRD90M) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 UCBrowser/11.3.8.976 U3/0.8.0 Mobile Safari/534.30"'
+    ].freeze
+  end
+end

data/lib/browze/client.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require 'down'
+require 'geocoder'
+require 'httparty'
+require 'fileutils'
+require 'tty-progressbar'
+
+module Browze
+  # Desktop and mobile clients inherit from the base client.
+  class Client
+    include HTTParty
+
+    attr_reader :cookies
+    attr_writer :headers
+
+    # Initialize the cookie jar and headers.
+    def initialize
+      @cookies = CookieHash.new
+      @headers = {}
+    end
+
+    # Perform a GET request to the given url.
+    #
+    # @param [String] url
+    # @return [Browze::Response]
+    def get(url)
+      resp = Browze::Response.new(self.class.get(url, headers: headers))
+      resp.set_cookie.each { |c| @cookies.add_cookies(c) }
+      resp
+    end
+
+    # Perform a POST request to the given url with a request body.
+    #
+    # @param [String] url
+    # @param [Hash] body_hash
+    # @return [Browze::Response]
+    def post(url, body_hash)
+      resp = Browze::Response.new(self.class.post(url, body: body_hash, headers: headers))
+      resp.set_cookie.each { |c| @cookies.add_cookies(c) }
+      resp
+    end
+
+    # Download a file with an optional filename.
+    #
+    # @param [String] url
+    # @param [String] filename
+    def download(url, filename: nil)
+      bar = TTY::ProgressBar.new('Downloading... [:bar] :percent')
+      tempfile = Down.download(
+        url,
+        content_length_proc: ->(content_length) { bar.update(total: content_length) },
+        progress_proc: ->(progress) { bar.current = progress }
+      )
+      filename ||= tempfile.original_filename
+      FileUtils.mv(tempfile, filename)
+      puts "Download complete: #{filename} (#{tempfile.size} bytes)"
+    end
+
+    # Get the updated headers.
+    #
+    # @return [Hash]
+    def headers
+      @headers.merge({ 'User-Agent' => user_agent, 'Cookie' => @cookies.to_cookie_string })
+    end
+
+    # Pick a random user agent and persist it in the instance.
+    #
+    # @return [String]
+    def user_agent
+      @user_agent ||= self.class::USER_AGENTS.sample
+    end
+
+    # Get the current public ip address.
+    #
+    # @return [String] the IP address
+    def ip
+      @ip ||= get('http://whatismyip.akamai.com').body
+    end
+
+    # Return the current location based on the public ip address.
+    #
+    # @return [String]
+    def location
+      l = Geocoder.search(ip).first
+      "#{l.city}, #{l.country}"
+    end
+  end
+end

data/lib/browze/response.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+
+module Browze
+  # Wrap the HTTParty response to assign custom attributes.
+  class Response
+    attr_reader :body,
+                :code,
+                :original
+
+    # Initialize the Response object with the original response data.
+    #
+    # @param [HTTParty::Response] response
+    def initialize(response)
+      @body = response.body
+      @code = response.code
+      @original = response
+    end
+
+    # Returns the response parsed with nokogiri.
+    #
+    # @return [Nokogiri::HTML4::Document]
+    def parsed
+      Nokogiri::HTML(body)
+    end
+
+    # Get the cookies in the set-cookie header field.
+    #
+    # @return [Array] the cookies to be set in the client headers.
+    def set_cookie
+      original.get_fields('Set-Cookie') || []
+    end
+  end
+end

data/lib/browze/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Browze
+  VERSION = '0.1.0'
+end

data/lib/browze.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'browze/version'
+require 'browze/client'
+require 'browze/client/mobile'
+require 'browze/client/desktop'
+require 'browze/response'
+
+module Browze
+  # Create a new Browze instance.
+  #
+  # @param [Symbol] type
+  # @return [Browze::Client]
+  def self.start(type = :desktop)
+    type == :mobile ? Browze::Client::Mobile.new : Browze::Client::Desktop.new
+  end
+end

metadata

@@ -0,0 +1,171 @@
+--- !ruby/object:Gem::Specification
+name: browze
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Gianpiero Addis
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2021-12-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: down
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: geocoder
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: tty-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-retry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
+email:
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- browze.gemspec
+- lib/browze.rb
+- lib/browze/client.rb
+- lib/browze/client/desktop.rb
+- lib/browze/client/mobile.rb
+- lib/browze/response.rb
+- lib/browze/version.rb
+homepage:
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key:
+specification_version: 4
+summary: A wrapper around HTTParty to quick start scraping tasks.
+test_files: []