pod_ident 1.0.8 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 91627e7d83c5598628aa8a1a677e61931e15b322f63158bbfabec7f20443bad7
-  data.tar.gz: 7b36f2fc9ee8f88bd23a623dfc49a767ebd8214cfcd8abe5b8acecc91722c3f9
+  metadata.gz: e198325ee4e089c59c75f1006e724fdd747178542003cb49806f2714bc3abd5e
+  data.tar.gz: ccae58509136c30d486261328ff800dbd7bf6502eaf872da32dfa89782fe7257
 SHA512:
-  metadata.gz: 45dee4cda059fd68082b9c5c9e4269ce52fd797aa5176a20f3708a0dd19da568ae27cbe85a8ab6826408be01aecf44ee6f3a84f9c3e81cecc9cfe9520bda5d1f
-  data.tar.gz: 245579bb696a18bd58ae234f9effdd801c24d404513ee9e33e6cfd05f80f8c68e6df60105372e9dc7092f8344d9c7941c297772ea13038663fe9c8c205d77840
+  metadata.gz: fdc493231340b1d3388e72495caae70b49a086625d2c26855023fd2b5302ed3be52ca708afb1cf69da3cad3007cd373231f96c21ddc425d6dabeea362f8ecbff
+  data.tar.gz: b68cbe2646ae9866ae4aa042d6b6a3b0e429588609112271961c2444748b4c95537cf52bbd5687f36904cf497f62de3d69982e4aa7e24d7fea9915e3c3be4a53

data/.rubocop.yml

@@ -16,5 +16,5 @@ Metrics/BlockLength:
     - 'Rakefile'
     - '**/*.rake'
     - 'spec/**/*.rb'
-Lint/UnneededCopDisableDirective:
+Lint/RedundantCopDisableDirective:
   Enabled: false

data/.ruby-version

@@ -1 +1 @@
-2.6.6
+3.2.2

data/Gemfile.lock

@@ -1,39 +1,39 @@
 PATH
   remote: .
   specs:
-    pod_ident (1.0.8)
+    pod_ident (1.1.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
     coderay (1.1.3)
-    diff-lcs (1.4.4)
+    diff-lcs (1.5.0)
     method_source (1.0.0)
-    pry (0.13.1)
+    pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    rake (13.0.1)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.2)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
+    rake (13.0.6)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.2)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.5)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.3)
-    rspec_junit_formatter (0.4.1)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
+    rspec_junit_formatter (0.6.0)
       rspec-core (>= 2, < 4, != 2.12.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.16)
+  bundler (~> 2.0)
   pod_ident!
   pry
   rake (~> 13.0)
@@ -41,4 +41,4 @@ DEPENDENCIES
   rspec_junit_formatter
 
 BUNDLED WITH
-   1.17.3
+   2.4.14

data/README.md

@@ -57,6 +57,21 @@ puts result.platform
 => 'iPod'
 ```
 
+### Bot detection
+
+
+```ruby
+require 'pod_ident'
+
+user_agent = 'Mozilla/5.0 (compatible; archive.org_bot http://archive.org/details/archive.org_bot)'
+result = PodIdent::Detector.detect(user_agent)
+
+puts result.app 
+=> 'Archive.org'
+puts result.bot? 
+=> true
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
@@ -67,6 +82,12 @@ The detection rules are found as a yaml file under `lib/detection_rules.yml`.
 
 Please, only edit this file, and afterwards run `bin/parse-rules` in order to generate both the detection rules as a `rb` file for production usage, and the complete rules with their corresponding test cases for testing.
 
+### Update bot detection rules
+
+For the bot detection rules we are using [ user-agents-v2](https://github.com/opawg/user-agents-v2/blob/master/src/bots.json), to import the json of this project and transform it into a yaml file readable by pod-ident you can run the following task: `rake update_bot_rules`.
+
+Once the yml file created run `bin/parse-rules` to generate all rb detection rules files for bots.
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/podigee/pod_ident.rb.

data/Rakefile

@@ -1,6 +1,61 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'json'
+require 'yaml'
+require 'net/http'
+require 'uri'
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec
+
+task :update_bot_rules do
+  BOT_JSON_SOURCE = 'https://raw.githubusercontent.com/opawg/user-agents-v2/master/src/bots.json'
+  BOT_JSON = File.expand_path('bots.json', __dir__)
+  BOT_YML = File.expand_path('lib/detection_rules_bots.yml', __dir__)
+
+  def update_bot_json
+    clean_json
+    uri = URI.parse(BOT_JSON_SOURCE)
+    response = Net::HTTP.get_response(uri)
+    File.open(BOT_JSON, 'w+') { |f| f.write(response.body) }
+  end
+
+  def clean_json
+    File.delete(BOT_JSON) if File.exist?(BOT_JSON)
+  end
+
+  def json_content
+    json = File.open(BOT_JSON).read
+    content = JSON.parse(json)
+  end
+
+  def generate_entry_yml(entry)
+    { 'app' => entry['name'],
+      'match' => { 'regex' => (entry['pattern']) },
+      'platform' => 'bot' }.merge(
+        entry_test_value(entry['examples'])
+      )
+  end
+
+  def entry_test_value(entry_examples)
+    return {} if entry_examples.nil?
+
+    examples = entry_examples.map do |example|
+      { 'userAgent' => example }
+    end
+
+    { 'test' => { 'userAgents' => examples } }
+  end
+
+  def write_yml_file(content)
+    File.open(BOT_YML, 'w+') { |f| f.write(content.to_yaml) }
+    clean_json
+  end
+
+  update_bot_json
+  all_entries = json_content['entries'].map do |entry|
+    generate_entry_yml(entry)
+  end
+  write_yml_file(all_entries)
+end

data/lib/detection_rules.yml

@@ -53,16 +53,20 @@
 - app: Apple Podcasts
   match:
     includes: watchOS
+    excluding:
+      regex: '^atc/|\\(null\\) watchOS/'
   platform:
     text: watchOS
   test:
     userAgents:
-      - userAgent: '(null)/(null) watchOS/5.0 model/Watch2,3 hwp/t8002 build/16R364 (6; dt:133)'
+      - userAgent: 'watchOS/5.0 model/Watch2,3 hwp/t8002 build/16R364 (6; dt:133)'
         platform: watchOS
 
 - app: Spotify
   match:
     startsWith: Spotify
+    excluding:
+      regex: '^Spotify/1\\.0$'
   platform:
     regexes:
       - '(Android|iOS)'
@@ -176,6 +180,8 @@
 - app: Overcast
   match:
     startsWith: Overcast
+    excluding:
+      regex: '^Overcast/1\\.0 Podcast Sync'
   platform:
     text: 'iOS'
   test:
@@ -429,6 +435,8 @@
 - app: Podbean
   match:
     startsWith: Podbean
+    excluding:
+      regex: "^Podbean Importer"
   platform:
     regex: '(iOS|Android)'
   test:
@@ -828,6 +836,8 @@
 - app: Twitter in-app browser
   match:
     includes: Twitter
+    excluding:
+      regex: "^TelegramBot "
   platform:
     regex: '(iPhone|iPad|Darwin|Android)'
     replacements:
@@ -946,6 +956,8 @@
 - app: Unknown client
   match:
     startsWith: '(null)'
+    excluding:
+      regex: '^atc/|\\(null\\) watchOS/'
   platform:
     regex: '(iPhone|iPad|iPod touch)'
   test:
@@ -966,6 +978,8 @@
 - app: Flipboard
   match:
     includes: Flipboard
+    excluding:
+      regex: FlipboardProxy/
   platform:
     regexes:
       - '(Android)'
@@ -979,8 +993,6 @@
         platform: Android
       - userAgent: Mozilla/5.0 (iPad; CPU OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Mobile/14E304 Flipboard/4.2.20
         platform: iPad
-      - userAgent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:49.0) Gecko/20100101 Firefox/49.0 (FlipboardProxy/1.2; +http://flipboard.com/browserproxy)
-        platform: Mac
 
 - app: iVoox
   match:
@@ -1081,3 +1093,13 @@
     userAgents:
       - userAgent: "sp-agent"
         platform: Android
+
+- app: RTL+
+  match:
+    startsWith: "RTL+"
+  platform:
+    regex: "(iOS|Android)"
+  test:
+    userAgents:
+      - userAgent: "RTL+/2.8.4.5-live-prod (Linux;Android 12) ExoPlayerLib/2.18.0"
+        platform: Android