ronin-web-user_agents 0.1.0.beta1

data/lib/ronin/web/user_agents/google_bot.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+#
+# ronin-web-user_agents - Yet another User-Agent string generator library.
+#
+# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# ronin-web-user_agents is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-web-user_agents is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-web-user_agents.  If not, see <https://www.gnu.org/licenses/>
+#
+
+require 'ronin/web/user_agents/chrome'
+
+module Ronin
+  module Web
+    module UserAgents
+      #
+      # Represents every possible `GoogleBot` `User-Agent` string.
+      #
+      module GoogleBot
+        # `GoogleBot` version
+        VERSION = '2.1'
+
+        # `GoogleBot' URL.
+        URL = 'http://www.google.com/bot.html'
+
+        # The Android version which `GoogleBot` Mobile uses.
+        ANDROID_VERSION = '6.0.1'
+
+        # The Android device which `GoogleBot` Mobile uses.
+        ANDROID_DEVICE = 'Nexus 5X Build/MMB29P'
+
+        #
+        # Builds a `GoogleBot` `User-Agent` string.
+        #
+        # @param [:search, :image, :video] crawler
+        #   The type of `GoogleBot`.
+        #
+        # @param [:desktop, :mobile, nil] compatible
+        #   Indicates whether to return a `(compatible: GoogleBot/...)`
+        #   `User-Agent` string.
+        #
+        # @param [String, nil] chrome_version
+        #   The optional Chrome version to include. Only is used when
+        #   `compatible: true` is given.
+        #
+        # @return [String]
+        #   The `GoogleBot` `User-Agent` string.
+        #
+        # @raise [ArgumentError]
+        #   An unsupported `crawler:` or `compatible:` value was given.
+        #
+        # @see https://developers.google.com/search/docs/advanced/crawling/overview-google-crawlers
+        #
+        # @api public
+        #
+        def self.build(crawler: :search, compatible: nil, chrome_version: nil)
+
+          case crawler
+          when :image
+            "Googlebot-Image/1.0"
+          when :video
+            "Googlebot-Video/1.0"
+          when :search
+            case compatible
+            when :desktop
+              googlebot = "GoogleBot/#{VERSION}; +#{URL}"
+
+              if chrome_version
+                "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; #{googlebot}) Chrome/#{chrome_version} Safari/537.36"
+              else
+                "Mozilla/5.0 (compatible; #{googlebot})"
+              end
+            when :mobile
+              unless chrome_version
+                raise(ArgumentError,"compatible: :mobile also requires the chrome_version: keyword argument")
+              end
+
+              "Mozilla/5.0 (Linux; Android #{ANDROID_VERSION}; #{ANDROID_DEVICE}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/#{chrome_version} Mobile Safari/537.36 (compatible; GoogleBot/#{VERSION}; +#{URL})"
+            when nil
+              "GoogleBot/#{VERSION} (+#{URL})"
+            else
+              raise(ArgumentError,"unsupported compatible: value (#{compatible.inspect})")
+            end
+          else
+            raise(ArgumentError,"unsupported crawler: value (#{crawler.inspect})")
+          end
+        end
+
+        # Supported `crawler:` values.
+        SUPPORTED_CRAWLERS = [:search, :image, :video]
+
+        # Supported `compatible:` values.
+        SUPPORTED_COMPATIBILITY_MODES = [:desktop, :mobile, nil]
+
+        # Known Chrome versions.
+        #
+        # @return [Array<String>]
+        KNOWN_CHROME_VERSIONS = Chrome::KNOWN_VERSIONS
+
+        #
+        # Builds a random `GoogleBot` `User-Agent` string.
+        #
+        # @param [:search, :image, :video] crawler
+        #   The type of `GoogleBot`.
+        #
+        # @param [:desktop, :mobile, nil] compatible
+        #   Indicates whether to return a `(compatible: GoogleBot/...)`
+        #   `User-Agent` string.
+        #
+        # @param [String, nil] chrome_version
+        #   The optional Chrome version to include.
+        #
+        # @return [String]
+        #   The `GoogleBot` `User-Agent` string.
+        #
+        # @api public
+        #
+        def self.random(crawler:        SUPPORTED_CRAWLERS.sample,
+                        compatible:     SUPPORTED_COMPATIBILITY_MODES.sample,
+                        chrome_version: KNOWN_CHROME_VERSIONS.sample)
+          build(
+            crawler:        crawler,
+            compatible:     compatible,
+            chrome_version: chrome_version
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ronin/web/user_agents/os/android.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+#
+# ronin-web-user_agents - Yet another User-Agent string generator library.
+#
+# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# ronin-web-user_agents is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-web-user_agents is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-web-user_agents.  If not, see <https://www.gnu.org/licenses/>
+#
+
+require 'ronin/web/user_agents/data_dir'
+
+module Ronin
+  module Web
+    module UserAgents
+      module OS
+        #
+        # Defines information about the Android Operating System (OS).
+        #
+        # @api private
+        #
+        module Android
+          # Known Android versions.
+          VERSIONS = %w[
+            10
+            10.0
+            11
+            11.0
+            12
+            12.1
+            4.0.4
+            4.1.2
+            4.2.2
+            4.4.2
+            4.4.4
+            5.0
+            5.0.2
+            5.1
+            5.1.1
+            6.0
+            6.0.1
+            7.0
+            7.1.1
+            7.1.2
+            8.0
+            8.0.0
+            8.1
+            8.1.0
+            9
+            9.0
+          ]
+
+          # Architectures supported by Android.
+          ARCHES = {
+            arm:   'arm',
+            arm64: 'arm_64',
+
+            nil => nil
+          }
+
+          # Known Android devices.
+          DEVICES = File.readlines(
+            File.join(DATA_DIR,'android','devices.txt'), chomp: true
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ronin/web/user_agents/os/linux.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+#
+# ronin-web-user_agents - Yet another User-Agent string generator library.
+#
+# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# ronin-web-user_agents is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-web-user_agents is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-web-user_agents.  If not, see <https://www.gnu.org/licenses/>
+#
+
+module Ronin
+  module Web
+    module UserAgents
+      module OS
+        #
+        # Defines information about the Linux Operating System (OS).
+        #
+        # @api private
+        #
+        module Linux
+          # Common Linux Distros.
+          DISTROS = {
+            ubuntu: 'Ubuntu',
+            fedora: 'Fedora',
+            arch:   'Arch',
+
+            nil => nil
+          }
+
+          # Architectures supported by Linux.
+          ARCHES = {
+            x86_64:  'x86_64',
+            aarch64: 'aarch64',
+            arm64:   'aarch64',
+            i686:    'i686',
+            x86:     'i686'
+          }
+        end
+      end
+    end
+  end
+end

data/lib/ronin/web/user_agents/os/mac_os.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+#
+# ronin-web-user_agents - Yet another User-Agent string generator library.
+#
+# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# ronin-web-user_agents is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-web-user_agents is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-web-user_agents.  If not, see <https://www.gnu.org/licenses/>
+#
+
+module Ronin
+  module Web
+    module UserAgents
+      module OS
+        #
+        # Defines information about the macOS Operating System (OS).
+        #
+        # @api private
+        #
+        module MacOS
+          # Known macOS versions.
+          VERSIONS = %w[
+            10.10
+            10.10.3
+            10.10.4
+            10.11.6
+            10.12.0
+            10.12.1
+            10.12.6
+            10.13.4
+            10.13.6
+            10.14.0
+            10.14.1
+            10.14.2
+            10.14.5
+            10.14.6
+            10.14.8
+            10.15.1
+            10.15.2
+            10.15.4
+            10.15.6
+            10.15.7
+            10.16
+            10.16.0
+            10.31.7
+            10.4
+            10.55
+            10.7.0
+            11.0.0
+            11.15
+            11.55
+            11.6.3
+            12.2.0
+            16.55
+          ]
+
+          # Known macOS versions, but in underscored notation.
+          VERSIONS_UNDERSCORED = {
+            '10.10'   => '10_10',
+            '10.10.3' => '10_10_3',
+            '10.10.4' => '10_10_4',
+            '10.11.6' => '10_11_6',
+            '10.12.0' => '10_12_0',
+            '10.12.1' => '10_12_1',
+            '10.12.6' => '10_12_6',
+            '10.13.4' => '10_13_4',
+            '10.13.6' => '10_13_6',
+            '10.14.0' => '10_14_0',
+            '10.14.1' => '10_14_1',
+            '10.14.2' => '10_14_2',
+            '10.14.5' => '10_14_5',
+            '10.14.6' => '10_14_6',
+            '10.14.8' => '10_14_8',
+            '10.15.1' => '10_15_1',
+            '10.15.2' => '10_15_2',
+            '10.15.4' => '10_15_4',
+            '10.15.6' => '10_15_6',
+            '10.15.7' => '10_15_7',
+            '10.16'   => '10_16',
+            '10.16.0' => '10_16_0',
+            '10.31.7' => '10_31_7',
+            '10.4'    => '10_4',
+            '10.55'   => '10_55',
+            '10.7.0'  => '10_7_0',
+            '11.0.0'  => '11_0_0',
+            '11.15'   => '11_15',
+            '11.55'   => '11_55',
+            '11.6.3'  => '11_6_3',
+            '12.2.0'  => '12_2_0',
+            '16.55'   => '16_55',
+          }
+          VERSIONS_UNDERSCORED.default_proc = ->(hash,version) {
+            version.tr('.','_')
+          }
+
+          # Architectures supported by macOS.
+          ARCHES = {
+            intel:  'Intel',
+            x86_64: 'Intel'
+          }
+
+        end
+      end
+    end
+  end
+end

data/lib/ronin/web/user_agents/os/windows.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+#
+# ronin-web-user_agents - Yet another User-Agent string generator library.
+#
+# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# ronin-web-user_agents is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-web-user_agents is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-web-user_agents.  If not, see <https://www.gnu.org/licenses/>
+#
+
+module Ronin
+  module Web
+    module UserAgents
+      module OS
+        #
+        # Defines information about the Windows Operating System (OS).
+        #
+        # @api private
+        #
+        module Windows
+          # Common Windows versions.
+          VERSIONS = {
+            xp:    '5.2',
+            vista: '6.0',
+            7   => '6.1',
+            8   => '6.2',
+            8.1 => '6.3',
+            10  => '10.0'
+          }
+
+          # Architectures supported by Windows.
+          ARCHES = {
+            wow64:  'WOW64',
+            win64:  'Win64; x64',
+            x86_64: 'Win64; x64',
+
+            nil => nil
+          }
+        end
+      end
+    end
+  end
+end

data/lib/ronin/web/user_agents/version.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+#
+# ronin-web-user_agents - Yet another User-Agent string generator library.
+#
+# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# ronin-web-user_agents is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-web-user_agents is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-web-user_agents.  If not, see <https://www.gnu.org/licenses/>
+#
+
+module Ronin
+  module Web
+    module UserAgents
+      # ronin-web-user_agents version
+      VERSION = '0.1.0.beta1'
+    end
+  end
+end

data/lib/ronin/web/user_agents.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+#
+# ronin-web-user_agents - Yet another User-Agent string generator library.
+#
+# Copyright (c) 2006-2022 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# ronin-web-user_agents is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ronin-web-user_agents is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with ronin-web-user_agents.  If not, see <https://www.gnu.org/licenses/>
+#
+
+# browsers
+require 'ronin/web/user_agents/chrome'
+require 'ronin/web/user_agents/firefox'
+
+# crawlers
+require 'ronin/web/user_agents/google_bot'
+
+require 'ronin/web/user_agents/version'
+
+module Ronin
+  module Web
+    #
+    # Provides categories of common `User-Agent` strings.
+    #
+    # ## Example
+    #
+    # Get a random `User-Agent` string:
+    #
+    #     user_agent = Ronin::Web::UserAgents.random
+    #     # => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.230 Safari/537.36"
+    #
+    # Get a random Chrome `User-Agent` string:
+    #
+    #     user_agent = Ronin::Web::UserAgents.chrome.random
+    #     # => "Mozilla/5.0 (Linux; Android 5.1.1; Redmi Note 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4688.3 Mobile Safari/537.36"
+    #
+    # Get a random Firefox `User-Agent` string:
+    #
+    #     user_agent = Ronin::Web::UserAgents.firefox.random
+    #     # => "Mozilla/5.0 (Windows NT 6.1; rv:78.0.2) Gecko/20100101 Firefox/78.0.2"
+    #
+    # Get a random `GoogleBot` `User-Agent` string:
+    #
+    #     user_agent = Ronin::Web::UserAgents.google_bot.random
+    #     # => "GoogleBot/2.1 (+http://www.google.com/bot.html)"
+    #
+    module UserAgents
+
+      #
+      # Google Chrome `User-Agent` strings.
+      #
+      # @return [Chrome]
+      #
+      # @example
+      #   user_agent = Ronin::Web::UserAgents.chrome.random
+      #   # => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.2495.20 Safari/537.36"
+      #
+      # @api public
+      #
+      def self.chrome
+        Chrome
+      end
+
+      #
+      # Alias for {chrome}.
+      #
+      # @return [Chrome]
+      #
+      # @see chrome
+      #
+      # @api public
+      #
+      def self.google_chrome
+        Chrome
+      end
+
+      #
+      # Firefox `User-Agent` strings.
+      #
+      # @return [Firefox]
+      #
+      # @example
+      #   user_agent = Ronin::Web::UserAgents.firefox.random
+      #   # => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16.0; rv:52.55.07) Gecko/20100101 Firefox/52.55.07" 
+      #
+      # @api public
+      #
+      def self.firefox
+        Firefox
+      end
+
+      #
+      # `GoogleBot` `User-Agent` strings.
+      #
+      # @return [GoogleBot]
+      #
+      # @example
+      #   user_agent = Ronin::Web::UserAgents.google_bot.random
+      #   # => "GoogleBot/2.1 (+http://www.google.com/bot.html)"
+      #
+      # @api public
+      #
+      def self.google_bot
+        GoogleBot
+      end
+
+      #
+      # Returns a random Browser `User-Agent` string from one of the following:
+      #
+      # * {chrome}
+      # * {firefox}
+      #
+      # @return [String, nil]
+      #   A random `User-Agent` string from the category.
+      #   Note, `nil` can be returned if the given block filter out all User
+      #   Agents.
+      #
+      # @example
+      #   Ronin::Web::UserAgents.random
+      #   # => "Mozilla/5.0 (X11; Fedora; Linux i686; en-IE; rv:123.4) Gecko/20100101 Firefox/123.4"
+      #
+      def self.random(&block)
+        method = [
+          :chrome,
+          :firefox,
+        ].sample
+
+        return send(method).random(&block)
+      end
+    end
+  end
+end

data/ronin-web-user_agents.gemspec

@@ -0,0 +1,61 @@
+# encoding: utf-8
+
+require 'yaml'
+
+Gem::Specification.new do |gem|
+  gemspec = YAML.load_file('gemspec.yml')
+
+  gem.name    = gemspec.fetch('name')
+  gem.version = gemspec.fetch('version') do
+                  lib_dir = File.join(File.dirname(__FILE__),'lib')
+                  $LOAD_PATH << lib_dir unless $LOAD_PATH.include?(lib_dir)
+
+                  require 'ronin/web/user_agents/version'
+                  Ronin::Web::UserAgents::VERSION
+                end
+
+  gem.summary     = gemspec['summary']
+  gem.description = gemspec['description']
+  gem.licenses    = Array(gemspec['license'])
+  gem.authors     = Array(gemspec['authors'])
+  gem.email       = gemspec['email']
+  gem.homepage    = gemspec['homepage']
+  gem.metadata    = gemspec['metadata'] if gemspec['metadata']
+
+  glob = lambda { |patterns| gem.files & Dir[*patterns] }
+
+  gem.files  = `git ls-files`.split($/)
+  gem.files  = glob[gemspec['files']] if gemspec['files']
+  gem.files += Array(gemspec['generated_files'])
+
+  gem.executables = gemspec.fetch('executables') do
+    glob['bin/*'].map { |path| File.basename(path) }
+  end
+
+  gem.extensions       = glob[gemspec['extensions'] || 'ext/**/extconf.rb']
+  gem.test_files       = glob[gemspec['test_files'] || 'spec/{**/}*_spec.rb']
+  gem.extra_rdoc_files = glob[gemspec['extra_doc_files'] || '*.{txt,md}']
+
+  gem.require_paths = Array(gemspec.fetch('require_paths') {
+    %w[ext lib].select { |dir| File.directory?(dir) }
+  })
+
+  gem.requirements              = gemspec['requirements']
+  gem.required_ruby_version     = gemspec['required_ruby_version']
+  gem.required_rubygems_version = gemspec['required_rubygems_version']
+  gem.post_install_message      = gemspec['post_install_message']
+
+  split = lambda { |string| string.split(/,\s*/) }
+
+  if gemspec['dependencies']
+    gemspec['dependencies'].each do |name,versions|
+      gem.add_dependency(name,split[versions])
+    end
+  end
+
+  if gemspec['development_dependencies']
+    gemspec['development_dependencies'].each do |name,versions|
+      gem.add_development_dependency(name,split[versions])
+    end
+  end
+end