queryable_logs 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 23b8ae959b8c34959d0db89adfcba5e50b38011264947b4de53f626bd4d1bb64
-  data.tar.gz: 1f9aeb0b1b2b5b8c40f0bb4245859ed06d72066f45abf6aac885f295cf0e6964
+  metadata.gz: 8022ed8aff78bfff25367198a3a95f95bfddb4b8c4596276e85634efe01cea67
+  data.tar.gz: 3f8d87a7bc9702831b6f0f4a4b0e4e90a0d025b680f422dbc4bf8cd4e5560958
 SHA512:
-  metadata.gz: 13c600df1ba652861169e0b9495dd1d98a71723d8e663a8b219a4c8ad872f04d906453e593cfc70e3b8ef7d8d927bdca53666c08fa268b0cd0637cf3736dcc41
-  data.tar.gz: 6cd4fb3c21e7e4e513e6df4a7aad840f34cb9512d5d79cc4d7004f83aa8896d89d66ad7ebb27dba3084e66421916c86c0e2ebbc856fd2062b84b2f3018db142a
+  metadata.gz: 6bd153ba202ae69f51fbbee6b6a3a5828c8c0c68321b8d4519ce08f526261a5035a2b49d64e8077cfc84e21a6f40f1d055d26e08d2ae25c387b65335b84ce9da
+  data.tar.gz: 0ccda5b146fdde3f8f9fa7276516213616b271840f76f69e6bd0f3c3fd92258d2a42b6e3bb9db3b6262a26682de172d3ca8f76c4625d2f255e80f34b0d4e59fd

data/Gemfile.lock

@@ -1,105 +1,31 @@
 PATH
   remote: .
   specs:
-    queryable_logs (0.1.0)
-      rails (>= 4)
+    queryable_logs (0.1.2)
+      activerecord (>= 4)
+      activesupport (>= 4)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (5.2.8.1)
-      actionpack (= 5.2.8.1)
-      nio4r (~> 2.0)
-      websocket-driver (>= 0.6.1)
-    actionmailer (5.2.8.1)
-      actionpack (= 5.2.8.1)
-      actionview (= 5.2.8.1)
-      activejob (= 5.2.8.1)
-      mail (~> 2.5, >= 2.5.4)
-      rails-dom-testing (~> 2.0)
-    actionpack (5.2.8.1)
-      actionview (= 5.2.8.1)
-      activesupport (= 5.2.8.1)
-      rack (~> 2.0, >= 2.0.8)
-      rack-test (>= 0.6.3)
-      rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    actionview (5.2.8.1)
-      activesupport (= 5.2.8.1)
-      builder (~> 3.1)
-      erubi (~> 1.4)
-      rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.0.3)
-    activejob (5.2.8.1)
-      activesupport (= 5.2.8.1)
-      globalid (>= 0.3.6)
     activemodel (5.2.8.1)
       activesupport (= 5.2.8.1)
     activerecord (5.2.8.1)
       activemodel (= 5.2.8.1)
       activesupport (= 5.2.8.1)
       arel (>= 9.0)
-    activestorage (5.2.8.1)
-      actionpack (= 5.2.8.1)
-      activerecord (= 5.2.8.1)
-      marcel (~> 1.0.0)
     activesupport (5.2.8.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
     arel (9.0.0)
-    builder (3.2.4)
     concurrent-ruby (1.2.2)
-    crass (1.0.6)
     diff-lcs (1.5.0)
-    erubi (1.12.0)
-    globalid (0.4.2)
-      activesupport (>= 4.2.0)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    loofah (2.21.1)
-      crass (~> 1.0.2)
-      nokogiri (>= 1.5.9)
-    mail (2.7.1)
-      mini_mime (>= 0.1.1)
-    marcel (1.0.2)
-    method_source (1.0.0)
-    mini_mime (1.1.2)
-    mini_portile2 (2.4.0)
     minitest (5.15.0)
-    nio4r (2.5.9)
-    nokogiri (1.10.10)
-      mini_portile2 (~> 2.4.0)
-    rack (2.2.8)
-    rack-test (2.1.0)
-      rack (>= 1.3)
-    rails (5.2.8.1)
-      actioncable (= 5.2.8.1)
-      actionmailer (= 5.2.8.1)
-      actionpack (= 5.2.8.1)
-      actionview (= 5.2.8.1)
-      activejob (= 5.2.8.1)
-      activemodel (= 5.2.8.1)
-      activerecord (= 5.2.8.1)
-      activestorage (= 5.2.8.1)
-      activesupport (= 5.2.8.1)
-      bundler (>= 1.3.0)
-      railties (= 5.2.8.1)
-      sprockets-rails (>= 2.0.0)
-    rails-dom-testing (2.1.1)
-      activesupport (>= 5.0.0)
-      minitest
-      nokogiri (>= 1.6)
-    rails-html-sanitizer (1.5.0)
-      loofah (~> 2.19, >= 2.19.1)
-    railties (5.2.8.1)
-      actionpack (= 5.2.8.1)
-      activesupport (= 5.2.8.1)
-      method_source
-      rake (>= 0.8.7)
-      thor (>= 0.19.0, < 2.0)
-    rake (10.5.0)
+    rake (13.2.1)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
       rspec-expectations (~> 3.12.0)
@@ -113,29 +39,18 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
-    sprockets (3.7.2)
-      concurrent-ruby (~> 1.0)
-      rack (> 1, < 3)
-    sprockets-rails (3.2.2)
-      actionpack (>= 4.0)
-      activesupport (>= 4.0)
-      sprockets (>= 3.0.0)
-    thor (1.2.2)
     thread_safe (0.3.6)
     tzinfo (1.2.11)
       thread_safe (~> 0.1)
-    websocket-driver (0.7.6)
-      websocket-extensions (>= 0.1.0)
-    websocket-extensions (0.1.5)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.17)
+  bundler (~> 2)
   queryable_logs!
-  rake (~> 10.0)
+  rake (>= 13.0)
   rspec (~> 3.0)
 
 BUNDLED WITH
-   1.17.3
+   2.6.7

data/README.md

@@ -21,12 +21,54 @@ Or install it yourself as:
 Run
     `rails g queryable_logs`.
 
-This will generate a migration file and an initializer file. queryable_logs also logs the current user id. Let the gem know which method you are using to get the current user. Default is set to `current_user`.
+This will generate a migration file.
+```ruby
+class CreateTrailLogs < ActiveRecord::Migration
+  def change
+    create_table :trail_logs do |t|
+      t.integer :user_id
+      t.string :ip_address
+      t.string :controller
+      t.string :action
+      t.string :format
+      t.string :http_verb
+      t.text :params_hash
+      t.datetime :logged_at
+      t.string :response_code
+      t.string :request_url
+      t.string :sig
+
+      t.timestamps null: false
+    end
+  end
+end
+```
+and an initializer file.
+```ruby
+class Trail
+  cattr_accessor :current_user_method, :logger, :saving_logs
+  LogFile = Rails.root.join('log', 'trail.log')
+  delegate :debug, :info, :warn, :error, :fatal, :to => :logger
+end
+
+Trail.logger = Logger.new(Trail::LogFile)
+Trail.logger.level = 'info' # could be debug, info, warn, error or fatal
+Trail.current_user_method = :current_user
+```
+queryable_logs also logs the current user id. Let the gem know which method you are using to get the current user. Default is set to `current_user`.
+
+Finally, include the `QueryableLogs::WriteLog` in the base controller, typically the `ApplicationController`.
+```ruby
+class ApplicationController < ActionController::Base
+  include QueryableLogs::WriteLog
+end
+```
+
+Enter the following task to your crontab `rake parse:logs_to_db`
+
+eg: `0 * * * * cd /Users/akshaytakkar/sample_rails_app && /Users/akshaytakkar/.rvm/wrappers/ruby-3.1.0/rake db:parse_log_and_save_trails >> /Users/akshaytakkar/sample_rails_app/log/worker.log 2>&1`
 
-Enter the following task to your crontab
-    `rake parse:logs_to_db`
-eg:
-    `* * * * * cd /Users/akshaytakkar/sample_rails_app && /Users/akshaytakkar/.rvm/wrappers/ruby-3.1.0/rake db:parse_log_and_save_trails >> /Users/akshaytakkar/sample_rails_app/log/worker.log 2>&1`
+This will run the `rake parse:logs_to_db` rake task every hour and log any errors or output from the task to `worker.log` file.
 
 ## Usage
 

data/lib/generators/queryable_logs_generator.rb

@@ -28,11 +28,6 @@ class QueryableLogsGenerator < Rails::Generators::Base
   end
 
   def self.next_migration_number(dirname)
-    if ActiveRecord::Base.timestamped_migrations
-      current_time = Time.now.utc
-      current_time.strftime("%Y%m%d%H%M%S")
-    else
-      "%.3d" % (current_numeric_version(dirname) + 1)
-    end
+    Time.now.utc.strftime("%Y%m%d%H%M%S")
   end
 end

data/lib/generators/templates/initializer.rb

@@ -1,10 +1,9 @@
 class Trail
   cattr_accessor :current_user_method, :logger, :saving_logs
-  LogFile = Rails.root.join('log', 'trail.log')
+  LogFile = Rails.root.join('log', "trail.log")
   delegate :debug, :info, :warn, :error, :fatal, :to => :logger
 end
 
-Trail.logger = Logger.new(Trail::LogFile)
+Trail.logger = Logger.new(Trail::LogFile, 'daily', 7)
 Trail.logger.level = 'info' # could be debug, info, warn, error or fatal
-Trail.current_user_method = :current_user
-Trail.saving_logs = false
+Trail.current_user_method = :current_user

data/lib/generators/templates/migration.rb

@@ -7,7 +7,7 @@ class CreateTrailLogs < ActiveRecord::Migration<%= migration_version %>
       t.string :action
       t.string :format
       t.string :http_verb
-      t.text :params_hash
+      t.json :params_hash
       t.datetime :logged_at
       t.string :response_code
       t.string :request_url

data/lib/generators/templates/task.rb

@@ -1,6 +1,8 @@
 namespace :db do
   desc "Parse trail log file and save to DB"
   task :parse_log_and_save_trails => :environment do
-    QueryableLogs::TrailLog.parse_log_and_save_trails
+    file_name = ENV["FILE_NAME"] || "trail.log"
+    b = Benchmark.measure { QueryableLogs::TrailLog.parse_log_and_save_trails(file_name) }
+    puts "Trail file #{file_name} processed at #{Time.zone.now.strftime('%Y-%m-%d %H:%M')} in: #{'%.4f' % b.total}s"
   end
 end

data/lib/queryable_logs/trail_log.rb

@@ -1,33 +1,14 @@
 module QueryableLogs
   class TrailLog < ActiveRecord::Base
-    serialize :params_hash, JSON
-
     def self.parse_log_and_save_trails(log_file_name="trail.log")
-      trails_recorded = []
-      lines = []
-      bytes_to_read = 0
       log_file_path = "#{Rails.root}/log/#{log_file_name}"
-      
-      begin
-        File.open(log_file_path, File::RDONLY) do |f|
-          f.flock(File::LOCK_SH)
-          bytes_to_read = File.size(log_file_path)
-          f.flock(File::LOCK_UN)
-        end
-      rescue
-        f.flock(File::LOCK_UN)
-        return
-      end
-
-      return if bytes_to_read == 0
-      
-      File.open(log_file_path) do |f|
-        lines = f.read(bytes_to_read).split("\n")
-      end
+      lines = []
+      File.open(log_file_path) { |f| f.each_line { |line| lines << line } }
       lines.each do |line|
         next if line.match(/\A# Logfile created on/) != nil
         if m = line.match(/^I, \[(.+?)\]  INFO -- : (.*)$/)
           request_time_and_pid, log_line = m[1..2]
+          next if TrailLog.where(sig: request_time_and_pid).count > 0
           log_line = log_line.sub(/ p:(.*)$/, '')
           log_params = {"p" => $1}
           log_params = log_params.merge(Hash[log_line.scan(/(\w+):(\S+)/)])
@@ -49,27 +30,7 @@ module QueryableLogs
                       logged_at: request_time)
         end
       end
-
-      if bytes_to_read > 0
-        log_file_ptr = nil
-        begin
-          File.open(log_file_path, "r+") do |f|
-            f.flock(File::LOCK_EX)
-            f.seek(bytes_to_read + 1)
-            fsize = File.size(log_file_path)
-            buffer = f.read(fsize - bytes_to_read)
-            f.seek(0)
-            f.write(buffer)
-            f.truncate(fsize - bytes_to_read)
-            f.flock(File::LOCK_UN)
-          end
-        rescue Exception => e
-          puts "msg = #{e.message}"
-          puts "backtrace = #{e.backtrace}"
-          log_file_ptr&.flock(File::LOCK_UN)
-          return
-        end
-      end
+      QueryableLogs::ParsedTrailLogFile.create(file_name: log_file_name)
     end
   end
 end

data/lib/queryable_logs/version.rb

@@ -1,3 +1,3 @@
 module QueryableLogs
-  VERSION = "0.1.0"
+  VERSION = "0.1.2"
 end

data/lib/queryable_logs/write_log.rb

@@ -9,7 +9,7 @@ module QueryableLogs
     def write_to_trail_log
       # vid, mid, ip, vrb, url, ctl, act, fmt, res, p (must be last)
       log_string = "uid:%{user_id} ip:%{ip_address} vrb:%{http_verb} url:%{url} ctl:%{controller} act:%{action} fmt:%{format} res:%{response_code} p:%{params_as_json}" % {
-        user_id: send(Trail.current_user_method).try(:id) || '',
+        user_id: self.respond_to?(Trail.current_user_method) ? send(Trail.current_user_method).try(:id) : '',
         ip_address: request.remote_ip,
         http_verb: request.request_method,
         url: Nokogiri::HTML(request.fullpath).text.strip,
@@ -19,19 +19,8 @@ module QueryableLogs
         params_as_json: Nokogiri::HTML(params.to_json).text.strip,
         response_code: response.code
       }
-
-      log_file_ptr = nil
-      begin
-        File.open(JSON.parse(Trail.logger.to_json)["logdev"]["filename"], File::RDWR) do |f|
-          log_file_ptr = f
-          f.flock(File::LOCK_EX)
-          Trail.logger.info(log_string)
-          f.flock(File::LOCK_UN)
-        end
-      rescue Exception => e
-        puts "something went wrong ->>>>> #{e.message} #{e.backtrace}"
-        puts "log_string = #{log_string}"
-        log_file_ptr&.flock(File::LOCK_UN)
+        
+      Trail.logger.info(log_string)
       end
     end
   end

data/queryable_logs.gemspec

@@ -36,8 +36,9 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "rails", ">= 4"
-  spec.add_development_dependency "bundler", "~> 1.17"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_runtime_dependency "activerecord", ">= 4"
+  spec.add_runtime_dependency "activesupport", ">= 4"
+  spec.add_development_dependency "bundler", "~> 2"
+  spec.add_development_dependency "rake", ">= 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
 end

metadata

@@ -1,17 +1,31 @@
 --- !ruby/object:Gem::Specification
 name: queryable_logs
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Akshay Takkar
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-10-10 00:00:00.000000000 Z
+date: 2025-04-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rails
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4'
+- !ruby/object:Gem::Dependency
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -30,28 +44,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.17'
+        version: '2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.17'
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -115,7 +129,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
+rubygems_version: 3.4.1
 signing_key:
 specification_version: 4
 summary: Make logs queryable, just like you would query any other activerecord table