trainspotter 0.1.0

data/app/models/trainspotter/request_record.rb

@@ -0,0 +1,133 @@
+module Trainspotter
+  class RequestRecord < Record
+    self.table_name = "requests"
+
+    belongs_to :session_record,
+               class_name: "Trainspotter::SessionRecord",
+               foreign_key: "session_id",
+               optional: true
+
+    scope :completed, -> { where(completed: true) }
+    scope :for_log_file, ->(log_file) { where(log_file: log_file) }
+
+    scope :recent, ->(log_file:, limit: nil) {
+      limit ||= Trainspotter.recent_request_limit
+      for_log_file(log_file).completed.order(started_at: :desc).limit(limit)
+    }
+
+    scope :since, ->(since_id) {
+      return all unless since_id
+
+      reference = find_by(id: since_id)
+      return none unless reference
+
+      where("created_at > ?", reference.created_at)
+    }
+
+    def self.poll_for_changes(log_file:, since_id: nil)
+      for_log_file(log_file)
+        .completed
+        .since(since_id)
+        .order(started_at: :asc)
+    end
+
+    def self.unique_ips(log_file:)
+      for_log_file(log_file)
+        .where.not(ip: nil)
+        .distinct
+        .pluck(:ip)
+        .sort
+    end
+
+    def self.upsert_from_request(log_file, request)
+      entries_data = request.entries.map do |entry|
+        {
+          raw: entry.raw,
+          type: entry.type,
+          timestamp: entry.timestamp&.iso8601,
+          metadata: entry.metadata
+        }
+      end
+
+      upsert(
+        {
+          log_request_id: request.id,
+          log_file: log_file,
+          method: request.method,
+          path: request.path,
+          status: request.status,
+          duration_ms: request.duration_ms,
+          ip: request.ip,
+          controller: request.controller,
+          action: request.action,
+          started_at: request.started_at,
+          entries_json: entries_data,
+          completed: request.completed?
+        },
+        unique_by: :log_request_id
+      )
+    end
+
+    def self.for_session(session_id, limit: 100)
+      where(session_id: session_id)
+        .completed
+        .order(started_at: :asc)
+        .limit(limit)
+    end
+
+    serialize :entries_json, coder: JSON
+
+    def entries
+      parsed_entries
+    end
+
+    def sql_entries
+      parsed_entries.select(&:sql?)
+    end
+
+    def render_entries
+      parsed_entries.select(&:render?)
+    end
+
+    private def parsed_entries
+      @parsed_entries ||= (entries_json || []).map do |data|
+        Ingest::Line.new(
+          raw: data["raw"],
+          type: data["type"]&.to_sym || :other,
+          timestamp: data["timestamp"] ? Time.parse(data["timestamp"]) : nil,
+          metadata: (data["metadata"] || {}).transform_keys(&:to_sym)
+        )
+      end
+    end
+
+    def sql_count
+      sql_entries.size
+    end
+
+    def sql_duration_ms
+      sql_entries.sum { |e| e.duration_ms || 0 }
+    end
+
+    def render_count
+      render_entries.size
+    end
+
+    def render_duration_ms
+      render_entries.sum { |e| e.duration_ms || 0 }
+    end
+
+    def status_class
+      case status
+      when 200..299 then "success"
+      when 300..399 then "redirect"
+      when 400..499 then "client-error"
+      when 500..599 then "server-error"
+      else "unknown"
+      end
+    end
+
+    def completed?
+      completed
+    end
+  end
+end

data/app/models/trainspotter/session_record.rb

@@ -0,0 +1,71 @@
+module Trainspotter
+  class SessionRecord < Record
+    self.table_name = "sessions"
+
+    has_many :request_records,
+             class_name: "Trainspotter::RequestRecord",
+             foreign_key: "session_id"
+
+    scope :for_log_file, ->(log_file) { where(log_file: log_file) }
+    scope :identified, -> { where.not(email: nil) }
+    scope :ongoing, -> { where(end_reason: "ongoing") }
+
+    scope :recent, ->(log_file:, include_anonymous: false, limit: 50) {
+      scope = for_log_file(log_file).order(started_at: :desc).limit(limit)
+      include_anonymous ? scope : scope.identified
+    }
+
+    def self.find_active(ip:, after:, log_file:)
+      where(ip: ip, log_file: log_file, end_reason: "ongoing")
+        .where("started_at > ?", after)
+        .order(started_at: :desc)
+        .first
+    end
+
+    def self.expire_before(cutoff, log_file:)
+      ongoing
+        .for_log_file(log_file)
+        .where("ended_at < ?", cutoff)
+        .update_all(end_reason: "timeout")
+    end
+
+    attribute :id, :string, default: -> { SecureRandom.hex(8) }
+
+    def anonymous?
+      email.nil?
+    end
+
+    def ongoing?
+      end_reason == "ongoing"
+    end
+
+    def time_range_display
+      return "Unknown" unless started_at
+      start_str = started_at.strftime("%b %d %H:%M")
+      end_str = ended_at&.strftime("%H:%M") || "now"
+      "#{start_str} - #{end_str}"
+    end
+
+    def duration_seconds
+      return nil unless started_at
+      end_time = ended_at || Time.current
+      (end_time - started_at).to_i
+    end
+
+    def duration_display
+      seconds = duration_seconds
+      return "Unknown" unless seconds
+
+      if seconds < 60
+        "#{seconds}s"
+      elsif seconds < 3600
+        minutes = seconds / 60
+        "#{minutes}m"
+      else
+        hours = seconds / 3600
+        minutes = (seconds % 3600) / 60
+        "#{hours}h #{minutes}m"
+      end
+    end
+  end
+end

data/app/views/layouts/trainspotter/application.html.erb

@@ -0,0 +1,20 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Trainspotter</title>
+  <%= csrf_meta_tags %>
+  <%= csp_meta_tag %>
+
+  <%= yield :head %>
+
+  <%= engine_stylesheet_link_tag "application" %>
+  <%= engine_javascript_importmap_tags "application", {
+    "@hotwired/stimulus" => "https://cdn.jsdelivr.net/npm/@hotwired/stimulus@3/+esm"
+  } %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/app/views/trainspotter/requests/_request.html.erb

@@ -0,0 +1,51 @@
+<details class="request-group <%= request.status_class %>" <%= "open" unless request.completed? %>>
+  <summary class="request-summary">
+    <span class="request-method method-<%= request.method.downcase %>"><%= request.method %></span>
+    <span class="request-path"><%= request.path %></span>
+    <% if request.controller %>
+      <span class="request-controller"><%= request.controller %>#<%= request.action %></span>
+    <% end %>
+    <span class="request-meta">
+      <% if request.status %>
+        <span class="request-status status-<%= request.status_class %>"><%= request.status %></span>
+      <% end %>
+      <% if request.duration_ms %>
+        <span class="request-duration"><%= "%.1f" % request.duration_ms %>ms</span>
+      <% end %>
+      <% if request.sql_count > 0 %>
+        <span class="request-sql" title="<%= request.sql_count %> queries, <%= "%.1f" % request.sql_duration_ms %>ms">
+          🗃️ <%= request.sql_count %>
+        </span>
+      <% end %>
+      <% if request.render_count > 0 %>
+        <span class="request-renders" title="<%= request.render_count %> templates, <%= "%.1f" % request.render_duration_ms %>ms">
+          📄 <%= request.render_count %>
+        </span>
+      <% end %>
+    </span>
+    <% if request.started_at %>
+      <time class="request-time" datetime="<%= request.started_at.iso8601 %>">
+        <%= request.started_at.strftime("%H:%M:%S") %>
+      </time>
+    <% end %>
+  </summary>
+
+  <div class="request-details">
+    <% request.entries.each do |entry| %>
+      <div class="log-entry entry-<%= entry.type %>">
+        <% case entry.type %>
+        <% when :sql %>
+          <span class="entry-badge sql">SQL</span>
+          <span class="entry-duration"><%= "%.2f" % entry.duration_ms %>ms</span>
+          <code class="entry-content"><%= ansi_to_html(entry.metadata[:query]) %></code>
+        <% when :render %>
+          <span class="entry-badge render">VIEW</span>
+          <span class="entry-duration"><%= "%.2f" % entry.duration_ms %>ms</span>
+          <span class="entry-content"><%= ansi_to_html(entry.metadata[:template]) %></span>
+        <% else %>
+          <pre class="entry-raw"><%= ansi_to_html(entry.raw) %></pre>
+        <% end %>
+      </div>
+    <% end %>
+  </div>
+</details>

data/app/views/trainspotter/requests/index.html.erb

@@ -0,0 +1,49 @@
+<div class="trainspotter"
+     data-controller="requests"
+     data-requests-poll-url-value="<%= poll_requests_path %>"
+     data-requests-root-url-value="<%= root_path %>"
+     data-requests-since-id-value="<%= @requests.first&.id %>"
+     data-requests-log-file-value="<%= @current_log_file %>"
+     data-requests-ip-value="<%= @current_ip %>">
+  <header class="trainspotter-header">
+    <h1>Trainspotter</h1>
+    <div class="trainspotter-controls">
+      <% if @available_log_files.size > 1 %>
+        <select name="log_file" class="log-file-selector" data-requests-target="logFile" data-action="change->requests#changeLogFile">
+          <% @available_log_files.each do |log_file| %>
+            <option value="<%= log_file %>" <%= "selected" if log_file == @current_log_file %>><%= log_file %></option>
+          <% end %>
+        </select>
+      <% else %>
+        <span class="current-log-file"><%= @current_log_file %></span>
+      <% end %>
+      <select name="ip" class="ip-filter-selector" data-requests-target="ipFilter" data-action="change->requests#changeIp">
+        <option value="">All IPs</option>
+        <% @available_ips.each do |ip| %>
+          <option value="<%= ip %>" <%= "selected" if ip == @current_ip %>><%= ip %></option>
+        <% end %>
+      </select>
+      <label class="auto-scroll-toggle">
+        <input type="checkbox" checked data-requests-target="autoScroll">
+        Auto-scroll
+      </label>
+      <span class="connection-status" data-requests-target="status">●</span>
+      <%= link_to "Sessions", sessions_path, class: "nav-link" %>
+    </div>
+  </header>
+
+  <main class="trainspotter-main">
+    <div class="request-list" data-requests-target="list">
+      <% @requests.each do |request| %>
+        <%= render "trainspotter/requests/request", request: request %>
+      <% end %>
+    </div>
+
+    <% if @requests.empty? %>
+      <div class="empty-state">
+        <p>No requests found in the log file.</p>
+        <p class="hint">Make some requests to your Rails app to see them here.</p>
+      </div>
+    <% end %>
+  </main>
+</div>

data/app/views/trainspotter/sessions/_session.html.erb

@@ -0,0 +1,28 @@
+<details class="session-group <%= session.ongoing? ? 'ongoing' : 'ended' %>" data-session-id="<%= session.id %>" data-action="toggle->sessions#loadSession">
+  <summary class="session-summary">
+    <span class="session-identity">
+      <% if session.anonymous? %>
+        <span class="session-email anonymous">Anonymous</span>
+      <% else %>
+        <span class="session-email"><%= session.email %></span>
+      <% end %>
+    </span>
+    <span class="session-ip"><%= session.ip %></span>
+    <span class="session-meta">
+      <span class="session-request-count" title="<%= session.request_count %> requests">
+        <%= session.request_count %> req
+      </span>
+      <span class="session-status <%= session.ongoing? ? 'ongoing' : session.end_reason %>">
+        <%= session.ongoing? ? 'Active' : session.end_reason.capitalize %>
+      </span>
+      <span class="session-duration"><%= session.duration_display %></span>
+    </span>
+    <time class="session-time" title="<%= session.time_range_display %>">
+      <%= session.started_at&.strftime("%H:%M:%S") || "Unknown" %>
+    </time>
+  </summary>
+
+  <div class="session-requests">
+    <p class="loading-hint">Loading requests...</p>
+  </div>
+</details>

data/app/views/trainspotter/sessions/index.html.erb

@@ -0,0 +1,42 @@
+<div class="trainspotter"
+     data-controller="sessions"
+     data-sessions-sessions-url-value="<%= sessions_path %>">
+  <header class="trainspotter-header">
+    <h1>Trainspotter - Sessions</h1>
+    <div class="trainspotter-controls">
+      <% if @available_log_files.size > 1 %>
+        <select name="log_file" class="log-file-selector" data-sessions-target="logFile" data-action="change->sessions#changeLogFile">
+          <% @available_log_files.each do |log_file| %>
+            <option value="<%= log_file %>" <%= "selected" if log_file == @current_log_file %>><%= log_file %></option>
+          <% end %>
+        </select>
+      <% else %>
+        <span class="current-log-file"><%= @current_log_file %></span>
+      <% end %>
+      <label class="show-anonymous-toggle">
+        <input type="checkbox" <%= "checked" if @show_anonymous %> data-sessions-target="showAnonymous" data-action="change->sessions#changeShowAnonymous">
+        Show anonymous
+      </label>
+      <%= link_to "Requests", requests_path, class: "nav-link" %>
+    </div>
+  </header>
+
+  <main class="trainspotter-main">
+    <div class="session-list">
+      <% @sessions.each do |session| %>
+        <%= render "trainspotter/sessions/session", session: session %>
+      <% end %>
+    </div>
+
+    <% if @sessions.empty? %>
+      <div class="empty-state">
+        <p>No sessions found in the log file.</p>
+        <% if !@show_anonymous %>
+          <p class="hint">Try checking "Show anonymous" to see sessions without identified users.</p>
+        <% else %>
+          <p class="hint">Make some requests to your Rails app to see sessions here.</p>
+        <% end %>
+      </div>
+    <% end %>
+  </main>
+</div>

data/config/cucumber.yml

@@ -0,0 +1,8 @@
+<%
+rerun = File.file?('rerun.txt') ? IO.read('rerun.txt') : ""
+rerun = rerun.strip.gsub /\s/, ' '
+rerun_opts = rerun.empty? ? "--format #{ENV['CUCUMBER_FORMAT'] || 'progress'} features" : "--format #{ENV['CUCUMBER_FORMAT'] || 'pretty'} #{rerun}"
+std_opts = "--format #{ENV['CUCUMBER_FORMAT'] || 'pretty'} --strict --tags 'not @wip' --publish-quiet"
+%>
+default: <%= std_opts %> features
+rerun: <%= rerun_opts %> --format rerun --out rerun.txt --strict --tags 'not @wip'

data/config/routes.rb

@@ -0,0 +1,15 @@
+Trainspotter::Engine.routes.draw do
+  root to: "requests#index"
+
+  resources :requests, only: [:index] do
+    collection do
+      get :poll
+    end
+  end
+
+  resources :sessions, only: [:index] do
+    member do
+      get :requests
+    end
+  end
+end

data/lib/trainspotter/background_worker.rb

@@ -0,0 +1,74 @@
+require "concurrent"
+
+module Trainspotter
+  class BackgroundWorker < Struct.new(:interval, :lock_path, :logger, :task_block, :timer_task, :lock_file, keyword_init: true)
+    class << self
+      attr_accessor :instance
+
+      def start(interval:, lock_path:, logger: nil, &task_block)
+        raise ArgumentError, "a block is required" unless task_block
+        return if instance&.running?
+
+        self.instance = new(interval:, lock_path:, logger:, task_block:)
+        instance.start
+      end
+
+      def stop
+        instance&.stop
+        self.instance = nil
+      end
+    end
+
+    def start
+      return if running?
+      return unless acquire_lock
+
+      self.timer_task = Concurrent::TimerTask.new(
+        execution_interval: interval,
+        run_now: true,
+        &task_block
+      )
+
+      timer_task.add_observer do |_time, _result, error|
+        if error
+          logger&.error "[Trainspotter] Background worker error: #{error.message}"
+          logger&.error error.backtrace.first(10).join("\n")
+        end
+      end
+
+      timer_task.execute
+
+      logger&.info "[Trainspotter] Background worker started (pid=#{Process.pid})"
+    end
+
+    def stop
+      timer_task&.shutdown
+      self.timer_task = nil
+      release_lock
+
+      logger&.info "[Trainspotter] Background worker stopped"
+    end
+
+    def running?
+      timer_task&.running?
+    end
+
+    private
+
+    def acquire_lock
+      self.lock_file = File.open(lock_path, File::RDWR | File::CREAT)
+      lock_file.flock(File::LOCK_EX | File::LOCK_NB)
+    rescue Errno::EWOULDBLOCK, Errno::EAGAIN
+      lock_file&.close
+      self.lock_file = nil
+      false
+    end
+
+    def release_lock
+      return unless lock_file
+      lock_file.flock(File::LOCK_UN)
+      lock_file.close
+      self.lock_file = nil
+    end
+  end
+end

data/lib/trainspotter/configuration.rb

@@ -0,0 +1,68 @@
+module Trainspotter
+  class Configuration
+    class << self
+      def setting(name, default: nil, &block)
+        attr_writer name
+
+        ivar = :"@#{name}"
+        define_method(name) do
+          unless instance_variable_defined?(ivar)
+            instance_variable_set(ivar, block ? instance_eval(&block) : default)
+          end
+          instance_variable_get(ivar)
+        end
+      end
+    end
+
+    setting(:log_directory) { Rails.root.join("log").to_s }
+    setting(:database_path) { Rails.root.join("tmp", "trainspotter.sqlite3").to_s }
+    setting(:session_timeout) { 30.minutes }
+    setting(:background_worker_interval) { 2.seconds }
+    setting :recent_request_limit, default: 100
+
+    setting(:filtered_paths) do
+      [
+        %r{^/assets/},
+        %r{^/packs/},
+        %r{^/vite/},
+        %r{^/rails/active_storage/},
+        %r{^/cable$},
+        %r{\.map$},
+        %r{\.hot-update\.}
+      ]
+    end
+
+    setting(:login_detectors) do
+      {
+        session_create: ->(request) {
+          return nil unless request.method == "POST" && request.path == "/session"
+          request.params&.dig("session", "email")
+        }
+      }
+    end
+
+    setting(:logout_detectors) do
+      {
+        session_destroy: ->(request) {
+          request.method == "DELETE" && request.path == "/session"
+        }
+      }
+    end
+
+    def available_log_files
+      Dir.glob(File.join(log_directory, "*.log")).map { |f| File.basename(f) }.sort
+    end
+
+    def default_log_file
+      "#{Rails.env}.log"
+    end
+
+    def filter_request?(path)
+      filtered_paths.any? { |pattern| pattern.match?(path) }
+    end
+
+    def internal_request?(request)
+      request.controller&.start_with?("Trainspotter::")
+    end
+  end
+end

data/lib/trainspotter/engine.rb

@@ -0,0 +1,45 @@
+module Trainspotter
+  class Engine < ::Rails::Engine
+    isolate_namespace Trainspotter
+    isolate_assets
+
+    initializer "trainspotter.silence_requests" do
+      config.after_initialize { Engine.silence_engine_requests }
+    end
+
+    initializer "trainspotter.background_worker" do
+      config.after_initialize do
+        Engine.start_background_worker unless Rails.env.test?
+      end
+
+      at_exit { Trainspotter::BackgroundWorker.stop }
+    end
+
+    def self.silence_engine_requests
+      if mount_path = engine_mount_path
+        Rails.application.config.middleware.insert_before(
+          Rails::Rack::Logger,
+          Rails::Rack::SilenceRequest,
+          path: %r{^#{Regexp.escape(mount_path)}}
+        )
+      end
+    end
+
+    def self.engine_mount_path
+      route = Rails.application.routes.routes.find do |r|
+        r.app.respond_to?(:app) && r.app.app == Trainspotter::Engine
+      end
+      route&.path&.spec&.to_s&.chomp("(.:format)")
+    end
+
+    def self.start_background_worker
+      Trainspotter::BackgroundWorker.start(
+        interval: Trainspotter.background_worker_interval,
+        lock_path: Rails.root.join("tmp", "trainspotter.lock"),
+        logger: Rails.logger
+      ) do
+        Trainspotter::IngestJob.new.perform
+      end
+    end
+  end
+end

data/lib/trainspotter/version.rb

@@ -0,0 +1,3 @@
+module Trainspotter
+  VERSION = "0.1.0"
+end

data/lib/trainspotter.rb

@@ -0,0 +1,30 @@
+require "trainspotter/version"
+require "trainspotter/configuration"
+require "isolate_assets"
+require "trainspotter/engine"
+require "trainspotter/background_worker"
+
+module Trainspotter
+
+  class << self
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configure
+      yield(configuration)
+    end
+
+    def method_missing(method, *args, &block)
+      if configuration.respond_to?(method)
+        configuration.public_send(method, *args, &block)
+      else
+        super
+      end
+    end
+
+    def respond_to_missing?(method, include_private = false)
+      configuration.respond_to?(method) || super
+    end
+  end
+end