browser_sense 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c7238bd65f27ab5515a8450831550d517d5c577d74d4d16d3a92a9cfab12598c
-  data.tar.gz: 6bd9cb1d1e1938ef1c10a863d6119ababbfc30eda63b2c68f28c3141397fc5b4
+  metadata.gz: e8fa590833475085fe473a90988cb128353267bc7bbcef259b0159a812fcf98c
+  data.tar.gz: 4b34a65d1875803994557840fb211e769fe406f63ccbd8b6b81b83f32216e382
 SHA512:
-  metadata.gz: 700b1052c5227ac87b7579223a337908b7ba520487c420d4387c366f29954672db9181370dac3cc988901df06a087c6c405e89619547d879ae0861cd1e785e00
-  data.tar.gz: 7873048616326259c8ccacd7ce35ca93698d1e9565557bb1afc67c97da4ddaea65941997bad7da893e81d5d55844f3a9aaa2c4d854c7ad4794c29835e25f1dfc
+  metadata.gz: 8cf8c94b32d56f262c414adbff5560f23da9a1e5263ff380583d8f81582a4f4fa35c0c198adfddd9633fe4902938fc2920b445c2183dc83df5ad60d61bc702af
+  data.tar.gz: 99d3a4e0c8938291aa0155459579dbdf83569dcecbf024d35de61d49ea771ab3105da1255c5698d7e4b6beca4711f28d42c87c699000229bf28a40f3a3fa972c

data/CHANGELOG.org

@@ -1,5 +1,22 @@
 #+TITLE: BrowserSense ChangeLog
 
+* [1.1.0] - <2025-12-21 Sun>
+
+- Store more information. The following fields are now appended to the lines
+  generated by BrowserSense:
+
+  - ip
+  - browser.version
+  - platform.version,
+  - browser.bot?
+  - browser.bot&.name
+  - browser.bot&.search_engine?
+
+- Use CSV to ensure the log line is properly parsable as a CSV string
+- Add the following two methods: =BrowserSenseParser.match?= and
+  =BrowserSenseParser.parse=
+
+
 * [1.0.1] - <2025-09-30 Tue>
 
 - Fixes documentation

data/README.org

@@ -47,23 +47,41 @@ following details:
 7. Hashed IP, which allows to track requests from the same IP, while preserving
    privacy
 8. Timestamp
+9. IP
+10. browser_version
+11. platform_version    
+12. bot?
+13. search_engine?
+14. bot name
 
 Information is stored in the application log, each line prefixed by
 =BrowserSense= and data presented in CSV.
 
 *Example*
 
+Before 1.10:
+
 #+begin_example bash
   $ cat production.log
   [...]
-  BrowserSense: "Firefox","linux","Unknown","Devise::SessionsController","new","12ca17b49af2289436f303e0166030a21e525d266e209267433801a8fd4071a0","2022-06-09T10:08:09+02:00"
+  [<production log header>] BrowserSense: "Firefox","linux","Unknown","Devise::SessionsController","new","12ca17b49af2289436f303e0166030a21e525d266e209267433801a8fd4071a0","2022-06-09T10:08:09+02:00"
   [...]
 #+end_example
 
+From 1.10 on (we add some field and use CSV for rendering the CSV, which removes unnecessary quotes).
+(Notice that the last field, =bot_name= is empty. Hence the final comma.
+
+#+begin_example bash
+$ cat production.log
+[...]
+[<production log header>] BrowserSense: Firefox,linux,Unknown,UsageController,index,html,12ca17b49af2289436f303e0166030a21e525d266e209267433801a8fd4071a0,2025-12-22T09:29:58+01:00,127.0.0.1,145,0,false,false,
+[...]
+#+end_example
+
 
 * Analyzing Data
 
-When you want to analyze data, extract the information with:
+*(Option 1)* When you want to analyze data, extract the information with:
 
 #+begin_example
 grep BrowserSense production.log | cut -f2- -d: > data.csv
@@ -71,12 +89,18 @@ grep BrowserSense production.log | cut -f2- -d: > data.csv
 
 and then, e.g., open the resulting file in a spreadsheet.
 
-There is also a ruby script =browser_sense=. The script takes as input the
-pathname of the file with the log and prints to stdout the result of various
+*(Option 2)* If you prefer you can use
+
+- =BrowserSenseParser.match?(line)= which returns true if the =line= is a line
+  produced by BrowserSense
+- =BrowserSenseParser.parse(line)= which returns a Hash with all the fields
+
+*(Option 3)* There is a Ruby script =browser_sense=. The script takes as input
+the pathname of the file with the log and prints to stdout the result of various
 analyses.  The script is a quick hack, wrapping =sed=, =grep= and =miller=
 ([[https://github.com/johnkerl/miller][Miller)]], which must be installed on your machine for the script to succeed.
 
-If you want more reports and plots, you can use the =log_sense= gem
+*(Option 4)* If you want more reports and plots, you can use the =log_sense= gem
 (https://rubygems.org/gems/log_sense), which understands the data generated by
 this gem.
         

data/app/controllers/browser_sense_filter.rb

@@ -1,5 +1,6 @@
 require "active_support/concern"
 require "browser"
+require "csv"
 
 module BrowserSenseFilter
   extend ActiveSupport::Concern
@@ -18,15 +19,27 @@ module BrowserSenseFilter
         now = DateTime.now
 
         logger = Rails.logger
+
         browser_data = [
-          b.name, b.platform, b.device.name,
-          controller.class.name, controller.action_name,
+          b.name,
+          b.platform,
+          b.device.name,
+          controller.class.name,
+          controller.action_name,
           request.format.symbol,
           hashed_ip,
-          now
+          now,
+          # added in v 1.1.10
+          ip,
+          b.version,
+          b.platform.version,
+          browser.bot?,
+          browser.bot&.search_engine?,
+          browser.bot&.name
         ]
 
-        browser_data_str = browser_data.map { |x| "\"#{x}\"" }.join(",")
+        CSV(browser_data_str = "")   { |csv_str| csv_str << browser_data }
+
         logger.info "BrowserSense: #{browser_data_str}"
       end
     end

data/app/services/browser_sense_parser.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module BrowserSenseParser
+  HEADERS = %i[
+    browser platform device_name controller method
+    request_format anon_ip timestamp
+    ip browser_version platform_version
+    bot search_engine bot_name
+  ]
+
+  def self.match?(line)
+    /Browser(Sense|Info):/.match? line
+  end
+
+  def self.parse(line)
+    begin
+      csv_portion = line.gsub(/^.*Browser(Sense|Info): /, "")
+      data = CSV.parse csv_portion, headers: HEADERS
+      log_id = log_id(line)
+      data.first.to_h.merge(log_id)
+    rescue => exception
+      Rails.logger.debug exception
+      {}
+    end
+  end
+
+  private_class_method def self.log_id(line)
+    matchdata = /-- : \[(?<log_id>[a-f0-9-]+)\]/.match(line)
+
+    { log_id: matchdata[:log_id] }
+  end
+end

data/browser_sense.gemspec

@@ -10,9 +10,10 @@ Gem::Specification.new do |spec|
 
   spec.summary = "Monitor which browser is accessing your Ruby on Rails app."
   spec.description = <<-EOS
-     BrowserSense logs information about the browsers accessing your RubyOnRails app.
+     BrowserSense logs information about the browsers using your RubyOnRails app.
 
-     Data can be easily extracted from the log, analyzed with the included script or with the log_sense gem.
+     Data can be easily extracted from the log, analyzed with the included
+     script or with the log_sense gem.
   EOS
   spec.homepage = "https://github.com/shair-tech/browser_sense"
   spec.license = "MIT"
@@ -36,8 +37,10 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  # Uncomment to register a new dependency of your gem
+  # We just log. All the recognition work is done by browser
   spec.add_dependency "browser", "~> 6.2.0"
+  # To properly quote log lines
+  spec.add_dependency "csv"
 
   # For more information and examples about making a new gem, check out our
   # guide at: https://bundler.io/guides/creating_gem.html

data/exe/browser_sense

@@ -22,7 +22,7 @@ end
 
 # Look for BrowserSense | cut info by Rails | remove lines with less than 7 fields
 # No need to remove double quotes with sed -e 's/\"//g'
-`grep -E -e 'Browser(Info|Sense)' #{LOG_FILE} | sed -E 's/.*Browser(Info|Sense): (.+,.+,.+,.+,.+,.+,.+,.+)/\\2/g' > #{TMP_FILE}`
+`grep -E -e 'Browser(Info|Sense)' #{LOG_FILE} | sed -E 's/.*Browser(Info|Sense): (.+,.+,.+,.+,.+,.+,.+,.+,(.+,.+,.+,.+,.+,.+)?)/\\2/g' > #{TMP_FILE}`
 
 # Look at the README file of BrowserSense for the field specification
 #
@@ -37,6 +37,15 @@ end
 # 7. Hashed IP, which allows to track requests from the same IP, while preserving privacy
 # 8. Timestamp
 #
+# Added in 1.10 (and made optional in the regexp above
+#
+#  9, ip
+# 10. b.version
+# 11. b.platform.version
+# 12. browser.bot?
+# 13. browser.bot&.search_engine?
+# 13. browser.bot&.name
+#
 
 # methods by os
 # the usage of reshape and unsparsify is still kind of magic to me.

data/lib/browser_sense/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module BrowserSense
-  VERSION = "1.0.1"
+  VERSION = "1.1.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: browser_sense
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - Adolfo Villafiorita
@@ -23,10 +23,25 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 6.2.0
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: |2
-       BrowserSense logs information about the browsers accessing your RubyOnRails app.
+       BrowserSense logs information about the browsers using your RubyOnRails app.
 
-       Data can be easily extracted from the log, analyzed with the included script or with the log_sense gem.
+       Data can be easily extracted from the log, analyzed with the included
+       script or with the log_sense gem.
 email:
 - adolfo@shair.tech
 executables:
@@ -42,6 +57,7 @@ files:
 - README.org
 - Rakefile
 - app/controllers/browser_sense_filter.rb
+- app/services/browser_sense_parser.rb
 - browser_sense.gemspec
 - exe/browser_sense
 - lib/browser_sense.rb