trainspotter 0.1.0

data/app/jobs/trainspotter/ingest/parser.rb

@@ -0,0 +1,194 @@
+module Trainspotter
+  module Ingest
+    class Parser
+      # Pattern to extract request ID tag from tagged logger output
+      # e.g., "[5de6cb4c-4a8e-4d87-bafd-3ce2281e26f4] Started GET..."
+      # or "  [req-id] Post Load (0.5ms)..." (tag after leading whitespace)
+      TAG_PATTERN = /^(?<leading_space>\s*)\[(?<request_id>[^\]]+)\]\s*/
+
+      # Regex patterns for Rails log formats
+      PATTERNS = {
+        # Started GET "/posts" for 127.0.0.1 at 2024-01-06 10:00:00 +0000
+        request_start: /^Started (?<method>GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS) "(?<path>[^"]+)" for (?<ip>[\d.]+) at (?<timestamp>.+)$/,
+
+        # Processing by PostsController#index as HTML
+        # Also handles namespaced controllers like Trainspotter::LogsController
+        processing: /^Processing by (?<controller>[\w:]+)#(?<action>\w+) as (?<format>\w+|\*\/\*)/,
+
+        # Parameters: {"session"=>{"email"=>"alice@example.com", "password"=>"[FILTERED]"}}
+        params: /^\s*Parameters: (?<params_string>.+)$/,
+
+        # Post Load (0.5ms)  SELECT "posts".* FROM "posts"
+        sql: /^\s*(?<name>[\w\s]+) \((?<duration>[\d.]+)ms\)\s+(?<query>.+)$/,
+
+        # Rendered posts/index.html.erb within layouts/application (Duration: 5.0ms | GC: 0.0ms)
+        render: /^\s*Rendered (?<template>[^\s]+)(?: within (?<layout>[^\s]+))? \(Duration: (?<duration>[\d.]+)ms/,
+
+        # Completed 200 OK in 50ms (Views: 40.0ms | ActiveRecord: 5.0ms | Allocations: 1234)
+        request_end: /^Completed (?<status>\d+) .+ in (?<duration>[\d.]+)ms/
+      }.freeze
+
+      def initialize
+        @groups_by_id = {}
+        @current_untagged_group = nil
+        @groups = []
+      end
+
+      def parse_line(line)
+        line = sanitize_encoding(line.chomp)
+        return nil if line.strip.empty?
+
+        request_id, content = extract_tag(line)
+        entry = identify_entry(content)
+
+        if request_id
+          handle_tagged_entry(request_id, entry)
+        else
+          handle_untagged_entry(entry)
+        end
+
+        entry
+      end
+
+      def parse_file(path, limit: nil)
+        reset_state
+
+        File.foreach(path).with_index do |line, index|
+          break if limit && index >= limit
+          parse_line(line)
+        end
+
+        finalize_all_groups
+        @groups
+      end
+
+      def parse_lines(lines)
+        reset_state
+
+        lines.each { |line| parse_line(line) }
+
+        finalize_all_groups
+        @groups
+      end
+
+      def groups
+        @groups.dup
+      end
+
+      private
+
+      def reset_state
+        @groups = []
+        @groups_by_id = {}
+        @current_untagged_group = nil
+      end
+
+      def extract_tag(line)
+        if (match = line.match(TAG_PATTERN))
+          leading_space = match[:leading_space] || ""
+          content = leading_space + line.sub(TAG_PATTERN, "")
+          [ match[:request_id], content ]
+        else
+          [ nil, line ]
+        end
+      end
+
+      def handle_tagged_entry(request_id, entry)
+        case entry.type
+        when :request_start
+          @groups_by_id[request_id] = Request.new(id: request_id)
+          @groups_by_id[request_id] << entry
+        when :request_end
+          if (group = @groups_by_id[request_id])
+            group << entry
+            group.completed = true
+            @groups << group
+            @groups_by_id.delete(request_id)
+          end
+        else
+          @groups_by_id[request_id]&.<<(entry)
+        end
+      end
+
+      def handle_untagged_entry(entry)
+        case entry.type
+        when :request_start
+          finalize_untagged_group
+          @current_untagged_group = Request.new
+          @current_untagged_group << entry
+        when :request_end
+          if @current_untagged_group
+            @current_untagged_group << entry
+            @current_untagged_group.completed = true
+            finalize_untagged_group
+          end
+        else
+          @current_untagged_group << entry if @current_untagged_group
+        end
+      end
+
+      def finalize_untagged_group
+        if @current_untagged_group&.entries&.any?
+          @groups << @current_untagged_group
+        end
+        @current_untagged_group = nil
+      end
+
+      def finalize_all_groups
+        finalize_untagged_group
+        @groups_by_id.each_value do |group|
+          @groups << group if group.entries.any?
+        end
+        @groups_by_id = {}
+      end
+
+      def identify_entry(line)
+        PATTERNS.each do |type, pattern|
+          if (match = line.match(pattern))
+            return build_entry(line, type, match)
+          end
+        end
+
+        Line.new(raw: line, type: :other)
+      end
+
+      def build_entry(line, type, match)
+        metadata = match.named_captures.transform_keys(&:to_sym)
+
+        case type
+        when :request_start
+          timestamp = parse_timestamp(metadata[:timestamp])
+          Line.new(raw: line, type: type, timestamp: timestamp, metadata: metadata)
+        when :processing
+          Line.new(raw: line, type: type, metadata: metadata)
+        when :params
+          params_hash = ParamsParser.parse(metadata[:params_string])
+          metadata[:params] = params_hash
+          Line.new(raw: line, type: type, metadata: metadata)
+        when :sql
+          metadata[:duration_ms] = metadata.delete(:duration).to_f
+          Line.new(raw: line, type: type, metadata: metadata)
+        when :render
+          metadata[:duration_ms] = metadata.delete(:duration).to_f
+          Line.new(raw: line, type: type, metadata: metadata)
+        when :request_end
+          metadata[:status] = metadata[:status].to_i
+          metadata[:duration_ms] = metadata.delete(:duration).to_f
+          Line.new(raw: line, type: type, metadata: metadata)
+        else
+          Line.new(raw: line, type: type, metadata: metadata)
+        end
+      end
+
+      def parse_timestamp(str)
+        Time.parse(str)
+      rescue ArgumentError
+        nil
+      end
+
+      def sanitize_encoding(str)
+        str.encode("UTF-8", invalid: :replace, undef: :replace, replace: "?")
+      end
+    end
+  end
+end

data/app/jobs/trainspotter/ingest/processor.rb

@@ -0,0 +1,70 @@
+module Trainspotter
+  module Ingest
+    class Processor < Struct.new(:log_path, :session_builder, :chunk_size, keyword_init: true)
+      DEFAULT_CHUNK_SIZE = 10_000
+
+      def self.call(log_paths, chunk_size: DEFAULT_CHUNK_SIZE)
+        session_builder = Ingest::SessionBuilder.new
+
+        log_paths.each do |log_path|
+          next unless File.exist?(log_path)
+          new(log_path:, session_builder:, chunk_size:).call
+        end
+      end
+
+      def call
+        if position = unread_position
+          process_chunk(position)
+          expire_stale_sessions
+        end
+      end
+
+      private
+
+      def log_filename
+        File.basename(log_path)
+      end
+
+      def unread_position
+        position = FilePositionRecord.get_position(log_filename)
+        file_size = File.size(log_path)
+
+        # Handle log rotation
+        position = 0 if file_size < position
+        return if file_size == position
+
+        position
+      end
+
+      def process_chunk(position)
+        new_position, lines = read_chunk(position)
+        return if lines.empty?
+
+        parse_and_persist(lines)
+        FilePositionRecord.update_position(log_filename, new_position)
+      end
+
+      def parse_and_persist(lines)
+        parser = Parser.new
+        lines.each { |line| parser.parse_line(line) }
+
+        parser.groups.each do |request|
+          RequestRecord.upsert_from_request(log_filename, request)
+          session_builder.process_request(request, log_filename) if request.completed?
+        end
+      end
+
+      def expire_stale_sessions
+        session_builder.expire_stale_sessions(log_filename)
+      end
+
+      def read_chunk(position)
+        File.open(log_path, "r") do |file|
+          file.seek(position)
+          lines = file.each_line.take(chunk_size || DEFAULT_CHUNK_SIZE).to_a
+          [file.pos, lines]
+        end
+      end
+    end
+  end
+end

data/app/jobs/trainspotter/ingest/reader.rb

@@ -0,0 +1,84 @@
+module Trainspotter
+  module Ingest
+    class Reader
+      attr_reader :path, :parser
+
+      def initialize(filename = nil)
+        @filename = filename || Trainspotter.default_log_file
+        @path = File.join(Trainspotter.log_directory, @filename)
+        @parser = Parser.new
+        @file_position = 0
+      end
+
+      def read_recent(limit: 100)
+        return [] unless File.exist?(path)
+
+        lines = tail_lines(limit * 20)
+        groups = parser.parse_lines(lines)
+        groups.last(limit)
+      end
+
+      def read_new_lines
+        return [] unless File.exist?(path)
+
+        current_size = File.size(path)
+
+        if current_size < @file_position
+          @file_position = 0
+        end
+
+        return [] if current_size == @file_position
+
+        new_lines = []
+        File.open(path, "r") do |file|
+          file.seek(@file_position)
+          new_lines = file.readlines
+          @file_position = file.pos
+        end
+
+        new_lines
+      end
+
+      def poll_for_changes
+        new_lines = read_new_lines
+        return [] if new_lines.empty?
+
+        new_groups = []
+        new_lines.each do |line|
+          parser.parse_line(line)
+          if parser.groups.any? && parser.groups.last.completed?
+            new_groups << parser.groups.last
+          end
+        end
+
+        new_groups
+      end
+
+      private
+
+      def tail_lines(count)
+        return [] unless File.exist?(path)
+
+        lines = []
+        File.open(path, "r") do |file|
+          file.seek(0, IO::SEEK_END)
+          buffer = ""
+          chunk_size = 8192
+
+          while lines.size < count && file.pos > 0
+            read_size = [ chunk_size, file.pos ].min
+            file.seek(-read_size, IO::SEEK_CUR)
+            chunk = file.read(read_size)
+            file.seek(-read_size, IO::SEEK_CUR)
+            buffer = chunk + buffer
+            lines = buffer.lines
+          end
+
+          @file_position = file.size
+        end
+
+        lines.last(count)
+      end
+    end
+  end
+end

data/app/jobs/trainspotter/ingest/session_builder.rb

@@ -0,0 +1,52 @@
+module Trainspotter
+  module Ingest
+    class SessionBuilder
+      def initialize(configuration: Trainspotter.configuration)
+        @configuration = configuration
+      end
+
+      def process_request(request, log_file)
+        return unless request.completed? && request.ip
+
+        session = find_or_create_session(request.ip, request.started_at, log_file)
+
+        if (email = detect_login(request))
+          session.update!(email: email)
+        end
+
+        if detect_logout(request)
+          session.update!(ended_at: request.started_at, end_reason: "logout")
+        end
+
+        RequestRecord.where(log_request_id: request.id).update_all(session_id: session.id)
+        session.increment!(:request_count)
+        session.update!(ended_at: request.started_at) if session.ongoing?
+      end
+
+      def expire_stale_sessions(log_file)
+        cutoff = Time.current - @configuration.session_timeout
+        SessionRecord.expire_before(cutoff, log_file: log_file)
+      end
+
+      private
+
+      def find_or_create_session(ip, timestamp, log_file)
+        timeout_cutoff = timestamp - @configuration.session_timeout
+        SessionRecord.find_active(ip: ip, after: timeout_cutoff, log_file: log_file) ||
+          SessionRecord.create!(ip: ip, started_at: timestamp, log_file: log_file)
+      end
+
+      def detect_login(request)
+        @configuration.login_detectors.each_value do |detector|
+          email = detector.call(request)
+          return email if email
+        end
+        nil
+      end
+
+      def detect_logout(request)
+        @configuration.logout_detectors.any? { |_, detector| detector.call(request) }
+      end
+    end
+  end
+end

data/app/jobs/trainspotter/ingest_job.rb

@@ -0,0 +1,10 @@
+module Trainspotter
+  class IngestJob
+    def perform
+      log_paths = Trainspotter.available_log_files.map do |f|
+        File.join(Trainspotter.log_directory, f)
+      end
+      Trainspotter::Ingest::Processor.call(log_paths)
+    end
+  end
+end

data/app/models/trainspotter/file_position_record.rb

@@ -0,0 +1,17 @@
+module Trainspotter
+  class FilePositionRecord < Record
+    self.table_name = "file_positions"
+    self.primary_key = "log_file"
+
+    def self.get_position(log_file)
+      find_by(log_file: log_file)&.position || 0
+    end
+
+    def self.update_position(log_file, position)
+      upsert(
+        { log_file: log_file, position: position, updated_at: Time.current },
+        unique_by: :log_file
+      )
+    end
+  end
+end

data/app/models/trainspotter/record.rb

@@ -0,0 +1,103 @@
+module Trainspotter
+  class Record < ActiveRecord::Base
+    self.abstract_class = true
+
+    SCHEMA_VERSION = 4
+
+    class << self
+      def ensure_connected
+        return if @connected
+
+        establish_connection(
+          adapter: "sqlite3",
+          database: Trainspotter.database_path,
+          pool: 5,
+          timeout: 5000
+        )
+        ensure_schema
+        @connected = true
+      end
+
+      def reset_connection!
+        return unless @connected
+
+        connection_pool.disconnect!
+        connection_handler.remove_connection_pool(name) rescue nil
+        @connected = false
+      end
+
+      private
+
+      def ensure_schema
+        if schema_version != SCHEMA_VERSION
+          connection.drop_table(:requests, if_exists: true)
+          connection.drop_table(:sessions, if_exists: true)
+          connection.drop_table(:file_positions, if_exists: true)
+        end
+
+        define_schema
+      end
+
+      def define_schema
+        connection.create_table :requests, if_not_exists: true do |t|
+          t.string :log_request_id, null: false
+          t.string :log_file, null: false
+          t.string :method
+          t.string :path
+          t.integer :status
+          t.float :duration_ms
+          t.string :ip
+          t.string :controller
+          t.string :action
+          t.text :params_json
+          t.datetime :started_at
+          t.text :entries_json
+          t.boolean :completed, default: false
+          t.string :session_id
+          t.datetime :created_at, default: -> { "CURRENT_TIMESTAMP" }
+
+          t.index :log_request_id, unique: true, name: "idx_requests_log_request_id", if_not_exists: true
+          t.index [:log_file, :started_at], order: { started_at: :desc }, name: "idx_requests_log_file_started_at", if_not_exists: true
+          t.index [:log_file, :created_at], name: "idx_requests_log_file_created_at", if_not_exists: true
+          t.index :session_id, name: "idx_requests_session_id", if_not_exists: true
+        end
+
+        connection.create_table :sessions, id: false, if_not_exists: true do |t|
+          t.string :id, null: false, primary_key: true
+          t.string :ip, null: false
+          t.string :email
+          t.datetime :started_at
+          t.datetime :ended_at
+          t.string :end_reason, default: "ongoing"
+          t.integer :request_count, default: 0
+          t.string :log_file, null: false
+
+          t.index [:ip, :started_at], order: { started_at: :desc }, name: "idx_sessions_ip_started_at", if_not_exists: true
+          t.index :log_file, name: "idx_sessions_log_file", if_not_exists: true
+        end
+
+        connection.create_table :file_positions, id: false, if_not_exists: true do |t|
+          t.string :log_file, null: false, primary_key: true
+          t.integer :position, default: 0
+          t.datetime :updated_at, default: -> { "CURRENT_TIMESTAMP" }
+        end
+
+        connection.create_table :schema_migrations, id: false, if_not_exists: true do |t|
+          t.string :version, null: false
+        end
+
+        unless schema_version == SCHEMA_VERSION
+          connection.execute("DELETE FROM schema_migrations")
+          quoted_version = connection.quote(SCHEMA_VERSION.to_s)
+          connection.execute("INSERT INTO schema_migrations (version) VALUES (#{quoted_version})")
+        end
+      end
+
+      def schema_version
+        connection.select_value("SELECT version FROM schema_migrations ORDER BY version DESC LIMIT 1")&.to_i
+      rescue ActiveRecord::StatementInvalid
+        nil
+      end
+    end
+  end
+end

data/app/models/trainspotter/request.rb

@@ -0,0 +1,108 @@
+module Trainspotter
+  class Request
+    attr_reader :id, :entries
+    attr_accessor :completed
+
+    def initialize(id: nil)
+      @id = id || SecureRandom.hex(8)
+      @entries = []
+      @completed = false
+    end
+
+    def <<(entry)
+      @entries << entry
+    end
+
+    def completed?
+      @completed
+    end
+
+    def method
+      start_entry&.metadata&.dig(:method) || "?"
+    end
+
+    def path
+      start_entry&.metadata&.dig(:path) || "?"
+    end
+
+    def ip
+      start_entry&.metadata&.dig(:ip)
+    end
+
+    def controller
+      processing_entry&.metadata&.dig(:controller)
+    end
+
+    def action
+      processing_entry&.metadata&.dig(:action)
+    end
+
+    def params
+      params_entry&.metadata&.dig(:params)
+    end
+
+    def status
+      end_entry&.metadata&.dig(:status)
+    end
+
+    def duration_ms
+      end_entry&.metadata&.dig(:duration_ms)
+    end
+
+    def started_at
+      start_entry&.timestamp
+    end
+
+    def sql_entries
+      @entries.select(&:sql?)
+    end
+
+    def render_entries
+      @entries.select(&:render?)
+    end
+
+    def sql_count
+      sql_entries.size
+    end
+
+    def sql_duration_ms
+      sql_entries.sum { |e| e.duration_ms || 0 }
+    end
+
+    def render_count
+      render_entries.size
+    end
+
+    def render_duration_ms
+      render_entries.sum { |e| e.duration_ms || 0 }
+    end
+
+    def status_class
+      case status
+      when 200..299 then "success"
+      when 300..399 then "redirect"
+      when 400..499 then "client-error"
+      when 500..599 then "server-error"
+      else "unknown"
+      end
+    end
+
+    private
+
+    def start_entry
+      @entries.find(&:request_start?)
+    end
+
+    def processing_entry
+      @entries.find(&:processing?)
+    end
+
+    def params_entry
+      @entries.find(&:params?)
+    end
+
+    def end_entry
+      @entries.find(&:request_end?)
+    end
+  end
+end