browserx 1.0.0

data/bots.yml

@@ -0,0 +1,327 @@
+---
+200pleasebot: 200PleaseBot
+360spider: 360Spider
+abot: CrawlDaddy, abot
+addthis: AddThis
+adldxbot: Microsoft Bing Ads
+admantx: ADmantX Platform Semantic Analyzer
+adsbot-google: Google Adwords
+adstxtcrawler: AdsTxtCrawler
+advbot: AdvBot
+ahrefsbot: Ahrefs backlinks research tool
+alexa: Alexa Crawler
+anderspink: AndersPinkBot
+apache-httpclient: Java http library
+apachebench: ApacheBench (ab)
+apis-google: APIs-Google
+appengine-google: Google App Engine
+applebot: Apple Bot
+appsignal: AppSignal Bot
+archive.org_bot: Internet Archive (archive.org)
+archiveteam archivebot: ArchiveTeam ArchiveBot
+ask jeeves: Ask Jeeves
+asynchttpclient: Java http and WebSocket client library
+awe.sm: Awe.sm URL expander
+baidu: Baidu
+barkrowler: Barkrowler
+bdcbot: Big Data Corp
+bingbot: Microsoft Bing
+bingpreview: Microsoft Bing preview
+bitlybot: bit.ly bot
+blekkobot: Blekkobot
+blexbot: BLEXBot (webmeup)
+bot@linkfluence.net: Linkfluence bot
+bubing: BUbiNG
+bufferbot: BufferBot
+buibui-checkbot: buibui
+butterfly: Topsy Labs
+buzzbot: Buzzbot
+buzztalk: buzztalk
+catchbot: CatchBot (catchbot.com)
+check_http: Nagios monitor
+checkly: Checkly
+chrome-lighthouse: Chrome-Lighthouse
+cipacrawler: CipaCrawler
+cliqzbot: Cliqzbot
+cloudflare: CloudFlare-AlwaysOnline
+cmradar/0.1: CMRadar/0.1
+coldfusion: ColdFusion http library
+commoncrawl: CCBot
+comodo ssl checker: COMODO SSL Checker
+comodo-webinspector-crawler: Comodo
+copypants: BotPants
+crowsnest: Crowsnest
+curabot: cura.yt
+curl: curl unix CLI http client
+dap/nethttp: DAP/NetHTTP
+datafeedwatch: DataFeedWatch
+datagnionbot: datagnion.com/bot.html
+datanyze: Datanyze
+daumoa: Korean portal and search engine indexing bot
+developers.google.com/+/web/snippet/: Google Plus
+diffbot: Diffbot
+digitalpersona fingerprint software: HP Fingerprint scanner
+domain re-animator bot: Domain Re-Animator Bot
+domainsbot: DomainsBot
+domaintunocrawler: DomainTuno
+dotbot: Dot Bot
+duckduckbot: Duck Duck Go
+elb-healthchecker: AWS ELB HealthChecker
+embedly: Embedly
+eoaagent: EOAAgent
+everyonesocialbot: EveryoneSocial
+evrinid: Evri bot
+exabot: Exalead's bot
+exaleadcloudview: ExaleadCloudView
+ez publish: eZ Publish Link Validator
+facebookexternalhit: Facebook Bot
+facebot: Facebook Bot
+feedburner: RSS bot
+feedfetcher-google: Google Feedfetcher
+findxbot: Findxbot
+flipboardproxy: FlipboardProxy
+friendfeedbot: FriendFeed
+fyrebot: Fyrebot
+garlik: GarlikCrawler
+genieo: Genieo Web filter bot
+germcrawler: GermCrawler
+getprismatic.com: getprismatic.com
+gigabot: Gigabot spider
+gimme60bot: Gimme60 (gimme60.com)
+gimmeusabot: Gimme60 (gimme60.com)
+go http package: Go http library
+go-http-client: Go http client
+google page speed insights: Google Page Speed Insights
+google web preview: Google Instant Previews crawler
+google-site-verification: Google Site Verification
+google-structured-data-testing-tool: Google Structured Data Testing Tool
+google-structureddatatestingtool: Google Structured Data Testing Tool
+google-xrawler: Google Shopping
+googlebot: Google Bot
+googleimageproxy: Google Image Proxy
+googlestackdrivermonitoring-uptimechecks: Google Stackdriver Monitoring - Uptime Checks
+grapeshotcrawler: GrapeshotCrawler
+gravitybot: Gravity Bot
+hatena::bookmark: Hatena::Bookmark
+heritrix: heritrix
+https://developers.google.com/+/web/snippet: Google+ Snippet Fetcher
+httrack: HTTrack
+hubspot: HubSpot
+ia_archiver: Internet Archive (WayBackMachine)
+icoreservice: iCoreService
+idmarch: idmarch.org/bot.html
+implisensebot: ImplisenseBot
+inagist: URL resolver
+insieve: Insieve Bot
+insitesbot: Insitesbot
+instapaper: Instapaper
+istellabot: IstellaBot
+jaunt: Jaunt - Java Web Scraping & JSON Querying
+jetslide: Jetslide
+jobseeker: jobseeker.com.au/bot.html
+jooble: Jooble
+js-kit: URL resolver
+kemvibot: Kemvi
+kimengi: Kimengi Bot
+knows.is: knows.is
+kojitsubot: Kojitsubot
+komodiabot: KomodiaBot
+kraken: kraken
+laconica: Laconica
+lijit crawler: Lijit
+linkdexbot: Linkdex Bot
+linkedinbot: LinkedIn
+linkscrawler: LinksCrawler
+linode: Linode Longview
+lipperhey: Lipperhey
+livelapbot: Livelapbot
+loadtimebot: Load Time Bot
+longurl: URL expander service
+ltx71: ltx71.com
+lumibot: Lumibot
+magpie-crawler: magpie-crawler
+mail.ru_bot: Mail.ru Bot
+mappydata: Mappy
+mastodon: Mastodon URL expander
+mauibot: MauiBot
+meanpathbot: meanpath
+mediapartners-google: Google Adsense bot
+megaindex.ru: MegaIndex
+memorybot: mignify.com/bot.html
+metauri: MetaURI
+mfe_expand: Mcafee spider
+mir web crawler: MIR web crawler
+mj12bot: Majestic-12 spider
+mojeekbot: Mojeek UK search crawler
+ms search 6.0 robot: MS Search 6.0 Robot
+msnbot-media: Microsoft media bot
+msnbot: Microsoft bot
+nerdybot: NerdyBot
+netcraft: Netcraft
+netstate: netEstate NE Crawler
+netvibes: Personalized dashboard bot
+netzcheckbot: netzcheck
+newrelicmonitor: NewRelic monitor
+newrelicpinger: NewRelicPinger
+newsme: newsme
+niki-bot: niki-bot
+ning: NING - Yet Another Twitter Swarmer
+nutch: Apache search spider
+openhosebot: OpenHoseBot
+orangebot: OrangeBot
+paessler: paessler.com - PRTG Network Monitor
+pagesinventory: pagesinventory.com
+panopta: Monitoring service
+paperlibot: PaperLi
+peerindex: peerindex
+percolatecrawler: PercolateCrawler
+perfectmarketkwtbot: PerfectMarket
+phantomjs: PhantomJS
+pingdom: Pingdom monitoring
+pinterest: Pinterest
+plukkie: botje.com/plukkie.htm
+pr-cy.ru: PR-CY.RU
+privacyawarebot: PrivacyAwareBot
+proximic: Proximic Spider
+psbot-page: Picsearch
+pu_in: Pu_iN Crawler
+publiclibraryarchive.org: publiclibraryarchive.org
+pycurl: Python http library
+python-httplib2: Python-httplib2
+python-requests: Python http library
+python-urllib: Python http library
+queryseeker: QuerySeekerSpider
+quick-crawler: Quick-Crawler
+quicklook: QuickLook
+re-animator: Domain Re-Animator Bot
+readability: Readability
+rebelmouse: RebelMouse
+redditbot: Reddit Bot
+relateiq: RelateIQ
+riddler: Riddler Bot
+rogerbot: SeoMoz spider
+rssmicro: RSS/Atom Feed Robot (rssmicro.com)
+scouturlmonitor: ScoutURLMonitor
+scrapy: Scrapy
+screaming frog seo spider: Screaming Frog SEO Spider
+searchmetricsbot: SearchmetricsBot
+semanticbot: Semanticbot
+semrushbot: SEO analysis bot
+seo-audit: seo-audit-check-bot
+seobilitybot: SeobilityBot
+seodiver: SEOdiver
+seokicks: SEOKicks
+seznambot: SeznamBot
+shopwiki: ShopWiki
+shortlinktranslate: Link shortener
+showyoubot: Showyou iOS app spider
+siege: Joe Dog Siege
+sistrix: SISTRIX
+sitecheck: SiteCheck sitecrawl
+siteuptime: Site monitoring services
+skypeuripreview: SkypeUriPreview
+slack-imgproxy: Slack Image Proxy
+slack-linkexpanding: Slack Link Expanding
+slack: Slack Link Expanding
+slackbot: Slackbot
+slurp: Yahoo spider
+smtbot: SimilarTech
+snapchat: Snapchat
+socialrank: SocialRankIOBot
+sogou: Chinese search engine
+spbot: OpenLinkProfiler
+spinn3r: Spinn3r aggregator
+sputnikbot: SputnikBot
+squider: Squider
+statuscake: StatusCake
+swiftbot: Swiftype Bot
+tangibleebot: TangibleeBot
+teeraid: TeeRaidBot
+test certificate info: C http library?
+the knowledge ai: Knowledge AI Bot
+tineye: TinEye Bot
+traackr: Traackr Bot
+trendictionbot: Trendiction Search
+trendsmap: Trendsmap Resolver
+turnitinbot: TurnitinBot
+tweetedtimes: The Tweeted Times
+tweetmemebot: TweetMeMe Crawler
+twikle: Social web search bot
+twitjobsearch: TwitJobSearch
+twitmunin: Twitmunin
+twitterbot: Twitter URL expander
+twurly: Twurly
+typhoeus: Typhoeus
+umbot: uberMetrics
+unwindfetch: Gnip
+updown: Updown.io monitor
+uptimerobot: Uptime Robot
+vagabondo: Vagabondo
+vb project: Visual Basic
+vigil: Vigil
+vkshare: VKontake Sharer
+voilabot: VoilaBot
+vrcrawler: Venture Radar
+wasalive-bot: Wasalive Bots
+watchsumo: WatchSumo
+wbsearchbot: Ware Bay Best Buys
+webceo: online-webceo-bot
+webscout: Webscout
+wesee: WeSEE
+wget: wget unix CLI http client
+whatsapp: WhatsApp
+wikido: WikiDo
+woorank: WooRank
+wordpress: WordPress spider
+woriobot: woriobot
+wormly: WormlyBot
+wotbox: Wotbox
+xenu link sleuth: Xenu Link Sleuth
+xing-contenttabreceiver: Xing bot
+xovibot: XoviBot
+yacybot: YaCy
+yahoo-ad-monitoring: Yahoo Ad monitoring
+yandex: Yandex
+yanga: Yanga WorldSearch Bot
+yeti: Naver Corp
+yourls: YOURLS
+zabbix: Zabbix
+zelist.ro: feed parser
+zibb: ZIBB spider
+zitebot: Zite
+zoombot: ZoomBot
+zoominfobot: ZoominfoBot
+zyborg: Zyborg
+
+# AI Crawlers
+# https://darkvisitors.com
+amazonbot: Amazon
+anthropic-ai: Anthropic-AI
+applebot: Apple
+bytespider: TikTok
+ccbot: Common Crawl
+chatgpt-user: ChatGPT
+claude-web: Anthropic-AI
+cohere-ai: Cohere
+diffbot: Diffbot
+facebookbot: Facebook
+google-extended: Google
+googleother: Google
+gptbot: ChatGPT
+omgili: Webz.io
+perplexitybot: Perplexity
+webz.io: Webz.io
+youbot: You.com
+
+# Generic lib user agents go here.
+httpie: HTTPie
+eventmachine httpclient: Ruby http library
+go 1.1 package http: Go 1.1 package http
+htmlparser: HTMLParser
+http_request2: HTTP_Request2
+httpclient: HTTPClient
+jakarta commons: Jakarta Commons HttpClient
+java: Generic Java http library
+libwww-perl: Perl client-server library
+lwp-trivial: Another Perl library
+ruby: Ruby

data/languages.yml

@@ -0,0 +1,113 @@
+"af": "Afrikaans"
+"sq": "Albanian"
+"eu": "Basque"
+"bg": "Bulgarian"
+"be": "Byelorussian"
+"ca": "Catalan"
+"zh": "Chinese"
+"zh-CN": "Chinese/China"
+"zh-TW": "Chinese/Taiwan"
+"zh-HK": "Chinese/Hong Kong"
+"zh-SG": "Chinese/singapore"
+"hr": "Croatian"
+"cs": "Czech"
+"da": "Danish"
+"nl": "Dutch"
+"nl-NL": "Dutch/Netherlands"
+"nl-BE": "Dutch/Belgium"
+"en": "English"
+"en-GB": "English/United Kingdom"
+"en-US": "English/United States"
+"en-AU": "English/Australian"
+"en-CA": "English/Canada"
+"en-NZ": "English/New Zealand"
+"en-IE": "English/Ireland"
+"en-ZA": "English/South Africa"
+"en-JM": "English/Jamaica"
+"en-BZ": "English/Belize"
+"en-TT": "English/Trinidad"
+"et": "Estonian"
+"fo": "Faeroese"
+"fa": "Farsi"
+"fi": "Finnish"
+"fr": "French"
+"fr-BE": "French/Belgium"
+"fr-FR": "French/France"
+"fr-CH": "French/Switzerland"
+"fr-CA": "French/Canada"
+"fr-LU": "French/Luxembourg"
+"gl": "Galician"
+"de": "German"
+"de-AT": "German/Austria"
+"de-DE": "German/Germany"
+"de-CH": "German/Switzerland"
+"de-LU": "German/Luxembourg"
+"de-LI": "German/Liechtenstein"
+"el": "Greek"
+"he": "Hebrew"
+"he-IL": "Hebrew/Israel"
+"hi": "Hindi"
+"hu": "Hungarian"
+"ie-EE": "Internet Explorer/Easter Egg"
+"is": "Icelandic"
+"id": "Indonesian"
+"in": "Indonesian"
+"ga": "Irish"
+"it": "Italian"
+"it-CH": "Italian/ Switzerland"
+"ja": "Japanese"
+"km": "Khmer"
+"km-KH": "Khmer/Cambodia"
+"ko": "Korean"
+"lv": "Latvian"
+"lt": "Lithuanian"
+"mk": "Macedonian"
+"ms": "Malaysian"
+"mt": "Maltese"
+"no": "Norwegian"
+"pl": "Polish"
+"pt": "Portuguese"
+"pt-BR": "Portuguese/Brazil"
+"rm": "Rhaeto-Romanic"
+"ro": "Romanian"
+"ro-MO": "Romanian/Moldavia"
+"ru": "Russian"
+"ru-MO": "Russian /Moldavia"
+"gd": "Scots Gaelic"
+"sr": "Serbian"
+"sk": "Slovack"
+"sl": "Slovenian"
+"sb": "Sorbian"
+"es": "Spanish"
+"es-DO": "Spanish"
+"es-AR": "Spanish/Argentina"
+"es-CO": "Spanish/Colombia"
+"es-MX": "Spanish/Mexico"
+"es-ES": "Spanish/Spain"
+"es-GT": "Spanish/Guatemala"
+"es-CR": "Spanish/Costa Rica"
+"es-PA": "Spanish/Panama"
+"es-VE": "Spanish/Venezuela"
+"es-PE": "Spanish/Peru"
+"es-EC": "Spanish/Ecuador"
+"es-CL": "Spanish/Chile"
+"es-UY": "Spanish/Uruguay"
+"es-PY": "Spanish/Paraguay"
+"es-BO": "Spanish/Bolivia"
+"es-SV": "Spanish/El salvador"
+"es-HN": "Spanish/Honduras"
+"es-NI": "Spanish/Nicaragua"
+"es-PR": "Spanish/Puerto Rico"
+"sx": "Sutu"
+"sv": "Swedish"
+"sv-SE": "Swedish/Sweden"
+"sv-FI": "Swedish/Finland"
+"ts": "Thai"
+"tn": "Tswana"
+"tr": "Turkish"
+"uk": "Ukrainian"
+"ur": "Urdu"
+"vi": "Vietnamese"
+"xh": "Xshosa"
+"ji": "Yiddish"
+"zu": "Zulu"

data/lib/browser/accept_language.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Browser
+  class AcceptLanguage
+    def self.languages
+      @languages ||= YAML.load_file(Browser.root.join("languages.yml"))
+    end
+
+    def self.parse(accept_language)
+      return [] unless accept_language
+
+      languages = accept_language.split(",")
+      languages.map! { |string| new(string.squeeze(" ").strip) }
+      languages.reject! { |al| al.quality.zero? }
+      languages.sort_by.with_index { |al, idx| [-al.quality, idx] }
+    end
+
+    attr_reader :part
+
+    def initialize(part)
+      @part = part
+    end
+
+    def full
+      @full ||= [code, region].compact.join("-")
+    end
+
+    def name
+      self.class.languages[full] || self.class.languages[code]
+    end
+
+    def code
+      @code ||= part[/\A([^-;]+)/, 1]&.downcase
+    end
+
+    def region
+      @region ||= part[/\A(?:.*?)-([^;-]+)/, 1]&.upcase
+    end
+
+    def quality
+      @quality ||= begin
+        Float(quality_value || 1.0)
+      rescue ArgumentError
+        0.1
+      end
+    end
+
+    private def quality_value
+      qvalue = part[/;q=([\d.]+)/, 1]
+      qvalue = "0.0" if /\A0\.0?\z/.match?(qvalue)
+      qvalue = qvalue.squeeze(".") if qvalue
+      qvalue
+    end
+  end
+end

data/lib/browser/action_controller.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "action_controller/railtie"
+
+module Browser
+  module ActionController
+    extend ActiveSupport::Concern
+
+    included do
+      helper_method(:browser) if respond_to?(:helper_method)
+    end
+
+    private def browser
+      @browser ||= Browser.new(
+        request.headers["User-Agent"],
+        accept_language: request.headers["Accept-Language"]
+      )
+    end
+  end
+end

data/lib/browser/aliases.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative "../browser"
+require "forwardable"
+
+module Browser
+  module Aliases
+    PLATFORM_ALIASES = %w[
+      adobe_air? android? blackberry? chrome_os? firefox_os? ios? ios_app?
+      ios_webview? linux? mac? windows10? windows7? windows8? windows8_1?
+      windows? windows_mobile? windows_phone? windows_rt?
+      windows_touchscreen_desktop? windows_vista? windows_wow64? windows_x64?
+      windows_x64_inclusive? windows_xp?
+    ].freeze
+
+    DEVICE_ALIASES = %w[
+      blackberry_playbook? console? ipad? iphone? ipod_touch? kindle?
+      kindle_fire? mobile? nintendo? nintendo_switch? nintendo_wii?
+      nintendo_wiiu? playbook? playstation3? playstation4? playstation?
+      playstation_vita? ps3? ps4? psp? psp_vita? silk? surface? tablet? tv?
+      vita? wii? wiiu? xbox? xbox_360? xbox_one?
+    ].freeze
+
+    def self.included(target)
+      target.class_eval do
+        extend Forwardable
+
+        def_delegators :platform, *PLATFORM_ALIASES
+        def_delegators :device, *DEVICE_ALIASES
+      end
+    end
+  end
+end

data/lib/browser/alipay.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Browser
+  class Alipay < Base
+    def id
+      :alipay
+    end
+
+    def name
+      "Alipay"
+    end
+
+    def full_version
+      ua[%r{(?:AlipayClient)/([\d.]+)}i, 1] || "0.0"
+    end
+
+    def match?
+      ua.match?(/AlipayClient/i)
+    end
+  end
+end