pod_ident 1.0.8 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 91627e7d83c5598628aa8a1a677e61931e15b322f63158bbfabec7f20443bad7
-  data.tar.gz: 7b36f2fc9ee8f88bd23a623dfc49a767ebd8214cfcd8abe5b8acecc91722c3f9
+  metadata.gz: 8a65623d1fa8d1ed68eee9872ec22ba7fac7aa9f0295f00c229319b08a09528e
+  data.tar.gz: 1e0dd3b229e27445de6786e785fd657c719c1aa97bf77755baf34d5f3f539a7f
 SHA512:
-  metadata.gz: 45dee4cda059fd68082b9c5c9e4269ce52fd797aa5176a20f3708a0dd19da568ae27cbe85a8ab6826408be01aecf44ee6f3a84f9c3e81cecc9cfe9520bda5d1f
-  data.tar.gz: 245579bb696a18bd58ae234f9effdd801c24d404513ee9e33e6cfd05f80f8c68e6df60105372e9dc7092f8344d9c7941c297772ea13038663fe9c8c205d77840
+  metadata.gz: bb5d27b7af088f9ef7bca27ed898dacf54c823fb35329b18f6da7bcd018ee5279bfa939a95fa1f715674e50544d0ccd3b01ecae546e42f12e6f9ec0bf371f409
+  data.tar.gz: ecea67a6709cc4f96e4f07429eaaec189ae12a5213cdab384e9d7a0b5677837f3876564ad2e2ffb40c7dc631d912f3bc4069e29ab2658d92b500f592755196df

data/.rubocop.yml

@@ -16,5 +16,5 @@ Metrics/BlockLength:
     - 'Rakefile'
     - '**/*.rake'
     - 'spec/**/*.rb'
-Lint/UnneededCopDisableDirective:
+Lint/RedundantCopDisableDirective:
   Enabled: false

data/.ruby-version

@@ -1 +1 @@
-2.6.6
+3.2.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pod_ident (1.0.8)
+    pod_ident (1.1.0)
 
 GEM
   remote: https://rubygems.org/
@@ -33,7 +33,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.16)
+  bundler (~> 2.3.20)
   pod_ident!
   pry
   rake (~> 13.0)
@@ -41,4 +41,4 @@ DEPENDENCIES
   rspec_junit_formatter
 
 BUNDLED WITH
-   1.17.3
+   2.3.20

data/README.md

@@ -57,6 +57,21 @@ puts result.platform
 => 'iPod'
 ```
 
+### Bot detection
+
+
+```ruby
+require 'pod_ident'
+
+user_agent = 'Mozilla/5.0 (compatible; archive.org_bot http://archive.org/details/archive.org_bot)'
+result = PodIdent::Detector.detect(user_agent)
+
+puts result.app 
+=> 'Archive.org'
+puts result.bot? 
+=> true
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
@@ -67,6 +82,12 @@ The detection rules are found as a yaml file under `lib/detection_rules.yml`.
 
 Please, only edit this file, and afterwards run `bin/parse-rules` in order to generate both the detection rules as a `rb` file for production usage, and the complete rules with their corresponding test cases for testing.
 
+### Update bot detection rules
+
+For the bot detection rules we are using [ user-agents-v2](https://github.com/opawg/user-agents-v2/blob/master/src/bots.json), to import the json of this project and transform it into a yaml file readable by pod-ident you can run the following task: `rake update_bot_rules`.
+
+Once the yml file created run `bin/parse-rules` to generate all rb detection rules files for bots.
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/podigee/pod_ident.rb.

data/Rakefile

@@ -1,6 +1,61 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'json'
+require 'yaml'
+require 'net/http'
+require 'uri'
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec
+
+task :update_bot_rules do
+  BOT_JSON_SOURCE = 'https://raw.githubusercontent.com/opawg/user-agents-v2/master/src/bots.json'
+  BOT_JSON = File.expand_path('bots.json', __dir__)
+  BOT_YML = File.expand_path('lib/detection_rules_bots.yml', __dir__)
+
+  def update_bot_json
+    clean_json
+    uri = URI.parse(BOT_JSON_SOURCE)
+    response = Net::HTTP.get_response(uri)
+    File.open(BOT_JSON, 'w+') { |f| f.write(response.body) }
+  end
+
+  def clean_json
+    File.delete(BOT_JSON) if File.exist?(BOT_JSON)
+  end
+
+  def json_content
+    json = File.open(BOT_JSON).read
+    content = JSON.parse(json)
+  end
+
+  def generate_entry_yml(entry)
+    { 'app' => entry['name'],
+      'match' => { 'regex' => (entry['pattern']) },
+      'platform' => 'bot' }.merge(
+        entry_test_value(entry['examples'])
+      )
+  end
+
+  def entry_test_value(entry_examples)
+    return {} if entry_examples.nil?
+
+    examples = entry_examples.map do |example|
+      { 'userAgent' => example }
+    end
+
+    { 'test' => { 'userAgents' => examples } }
+  end
+
+  def write_yml_file(content)
+    File.open(BOT_YML, 'w+') { |f| f.write(content.to_yaml) }
+    clean_json
+  end
+
+  update_bot_json
+  all_entries = json_content['entries'].map do |entry|
+    generate_entry_yml(entry)
+  end
+  write_yml_file(all_entries)
+end

data/lib/detection_rules.yml

@@ -53,16 +53,20 @@
 - app: Apple Podcasts
   match:
     includes: watchOS
+    excluding:
+      regex: '^atc/|\\(null\\) watchOS/'
   platform:
     text: watchOS
   test:
     userAgents:
-      - userAgent: '(null)/(null) watchOS/5.0 model/Watch2,3 hwp/t8002 build/16R364 (6; dt:133)'
+      - userAgent: 'watchOS/5.0 model/Watch2,3 hwp/t8002 build/16R364 (6; dt:133)'
         platform: watchOS
 
 - app: Spotify
   match:
     startsWith: Spotify
+    excluding:
+      regex: '^Spotify/1\\.0$'
   platform:
     regexes:
       - '(Android|iOS)'
@@ -176,6 +180,8 @@
 - app: Overcast
   match:
     startsWith: Overcast
+    excluding:
+      regex: '^Overcast/1\\.0 Podcast Sync'
   platform:
     text: 'iOS'
   test:
@@ -429,6 +435,8 @@
 - app: Podbean
   match:
     startsWith: Podbean
+    excluding:
+      regex: "^Podbean Importer"
   platform:
     regex: '(iOS|Android)'
   test:
@@ -828,6 +836,8 @@
 - app: Twitter in-app browser
   match:
     includes: Twitter
+    excluding:
+      regex: "^TelegramBot "
   platform:
     regex: '(iPhone|iPad|Darwin|Android)'
     replacements:
@@ -946,6 +956,8 @@
 - app: Unknown client
   match:
     startsWith: '(null)'
+    excluding:
+      regex: '^atc/|\\(null\\) watchOS/'
   platform:
     regex: '(iPhone|iPad|iPod touch)'
   test:
@@ -966,6 +978,8 @@
 - app: Flipboard
   match:
     includes: Flipboard
+    excluding:
+      regex: FlipboardProxy/
   platform:
     regexes:
       - '(Android)'
@@ -979,8 +993,6 @@
         platform: Android
       - userAgent: Mozilla/5.0 (iPad; CPU OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Mobile/14E304 Flipboard/4.2.20
         platform: iPad
-      - userAgent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:49.0) Gecko/20100101 Firefox/49.0 (FlipboardProxy/1.2; +http://flipboard.com/browserproxy)
-        platform: Mac
 
 - app: iVoox
   match: