rail_stat_generator 0.1.0

data/templates/app/views/rail_stat/other.rhtml

@@ -0,0 +1,51 @@
+<h1>Other statistics on <%%= @subdomain %> </h1>
+<%%=render_partial 'menu'%>
+<h2>Flash supported</h2>
+<div id="path_stats_flash">
+  <table class="list">
+    <tr><th width='50%'>Version</th><th>Percentage</th></tr>
+    <%% initListClass
+       for data in @flashes %>
+    <tr class="<%%=popListClass%>"><td><%%=path_version_data(data['flash'])%></td><td><%%=data['percent']%> %</td></tr>
+    <%% end %>
+  </table>
+</div>
+
+<h2>JavaVM Enabled</h2>
+<table class="list">
+  <tr><th width='50%'>Version</th><th>Percentage</th></tr>
+  <%% initListClass
+     for data in @jes %>
+  <tr class="<%%=popListClass%>"><td><%%=path_version_data(data['java_enabled'])%></td><td><%%=data['percent']%> %</td></tr>
+  <%% end %>
+</table>
+
+<h2>Javascript suported</h2>
+<table class="list">
+  <tr><th width='50%'>JavaScript</th><th>Percentage</th></tr>
+  <%% initListClass
+     for data in @javas %>
+  <tr class="<%%=popListClass%>"><td><%%=path_version_data(data['java'])%></td><td><%%=data['percent']%> %</td></tr>
+  <%% end %>
+</table>
+
+<h2>Screen width</h2>
+<table class="list">
+  <tr><th width='50%'>Width</th><th>Percentage</th></tr>
+  <%% initListClass
+     for data in @widths %>
+  <tr class="<%%=popListClass%>"><td><%%=path_version_data(data['screen_size'])%></td><td><%%=data['percent']%> %</td></tr>
+  <%% end %>
+</table>
+
+<h2>Color depth</h2>
+<table class="list">
+  <tr><th width='50%'>Color depth</th><th>Percentage</th></tr>
+  <%% initListClass
+     for data in @colors %>
+  <tr class="<%%=popListClass%>"><td><%%=path_version_data(data['colors'])%></td><td><%%=data['percent']%> %</td></tr>
+  <%% end %>
+</table>
+
+</div>
+</div>

data/templates/app/views/rail_stat/path.rhtml

@@ -0,0 +1,81 @@
+<h1>Path statistics for <%%= @subdomain %> </h1>
+<%%=render_partial 'menu'%>
+<h2>Last paths</h2>
+<div id="path_stats">
+  Number of hits:&nbsp;
+  <a href='?nh=25&ise=<%%=@include_search_engines%>'>25</a>&nbsp;
+  <a href='?nh=50&ise=<%%=@include_search_engines%>'>50</a>&nbsp;
+  <a href='?nh=100&ise=<%%=@include_search_engines%>'>100</a>&nbsp;
+  <a href='?nh=150&ise=<%%=@include_search_engines%>'>150</a>&nbsp;
+  <a href='?nh=200&ise=<%%=@include_search_engines%>'>200</a>&nbsp;
+  <a href='?nh=250&ise=<%%=@include_search_engines%>'>250</a>&nbsp; 
+  <br/>
+  Show search engines: &nbsp;
+  <a href='?nh=<%%=@number_hits%>&ise=1'>On</a>&nbsp;
+  <a href='?nh=<%%=@number_hits%>&ise=0'>Off</a>&nbsp;  <br/>
+  <%% if @paths.size > 0 %>
+  <table class="list" width='100%'>
+    <tr>
+      <th>D/T</th>
+      <th>Path</th>
+      <th>Addr.</th>
+      <th>Browser</th>
+      <th>Platform</th>
+    </tr>
+    <%%  initListClass
+           for i in 0..@paths.size-1
+             row = @paths[i]
+             if not(row.marked?)
+               row.mark 
+               domain, refferer = "-", '' %>
+    <tr class='<%%=popListClass%>'> 
+      <td width='60px'><div class='cutField'><%%=get_time(row.dt)%></div></td>
+      <td width='200px'><div class='cutField'>
+          <%%       last_access_time = row.dt 
+                               # work through loop backwards so we get the *first* page time wise
+                               (@paths.size-1).downto(i+1) { |j|
+                if not(@paths[j].marked?) and row.remote_ip == @paths[j].remote_ip and (last_access_time - 900 <= @paths[j].dt)
+                  domain, refferer = @paths[j].domain, @paths[j].referer if domain == "-" and @paths[j].domain and not(@paths[j].domain == SITE_NAME)%>
+                   <%%=get_path_image(@paths[j].resource, @ordered_resources)%>
+                   <%%           @paths[j].mark
+                  last_access_time = @paths[j].dt 
+                end
+              }
+                               # add last page %>
+                   <%%=get_path_image(row.resource, @ordered_resources)%>  
+                   <%%       domain, refferer = row.domain, row.referer if domain == "-" and row.domain and not(row.domain == SITE_NAME) %>   
+                   <%% if domain and not(domain.nil?) and not(domain == '-') %>
+                   from &nbsp; <a href="<%%=refferer%>"><%%=domain%></a>
+                   <%% end %>   
+      </div></td>
+      <td><div class='cutField'><%%=get_domain_name(row.remote_ip)%></div></td>
+      <td width='50px'><div class='cutField'><%%=row.browser%>
+          <%%  unless row.version =="unknown" %>
+          &nbsp;<%%=row.version%>
+          <%%  end %>
+      </div></td>
+      <td width='50px'><div class='cutField'><%%=row.platform%></div></td>
+    </tr>
+    <%%    end 
+          end %>
+  </table>
+  <%% end %>
+</div>
+
+<div id="pathslegend">
+  <h2>Legend</h2>
+  <%% if @orarr.size > 0 %>
+          <table class="list">
+            <tr><th colspan="2">Resource</th><th>Hits statistics</th></tr>
+            <%%   initListClass
+          for resource in @orarr %>
+            <tr class="<%%=popListClass%>">
+    <td><%%=get_path_image(resource['resource'], @ordered_resources)%></td>
+    <td><a href='<%%=resource['resource']%>'><%%=h resource['resource'] %></a></td>
+    <td><%%=resource['requests']%></td></tr>
+            <%%   end %>
+          </table>
+          <%% end %>
+        </div>
+</div>
+</div>

data/templates/app/views/rail_stat/platform.rhtml

@@ -0,0 +1,25 @@
+<h1>Platform statistics for <%%= @subdomain %> </h1>
+<%%=render_partial 'menu'%>
+<h2>Operating systems</h2>
+<div id="path_stats_pl">
+  <table class="list">
+    <tr><th>OS</th><th>%</th></tr>
+    <%% initListClass
+   for pl in @platforms %>
+    <tr class="<%%=popListClass%>"><td><%%=pl['platform']%></td><td><%%=pl['percent']%>%</td></tr>
+    <%% end %>
+  </table>
+</div>
+
+<h2>Browsers usage</h2>
+<div id="path_stats_br">
+  <table class="list">
+    <tr><th>Browser</th><th>Version</th><th>%</th></tr>
+    <%% initListClass
+       for br in @browsers %>
+    <tr class="<%%=popListClass%>"><td><%%=br['browser']%></td><td><%%=br['version']%></td><td><%%=br['percent']%>%</td></tr>
+    <%% end %>
+  </table>
+</div>
+</div>
+</div>

data/templates/app/views/rail_stat/refs.rhtml

@@ -0,0 +1,25 @@
+<h1>Referrers statistic for <%%= @subdomain %> </h1>
+<%%=render_partial 'menu'%>
+<h2>Repeat referrers</h2>
+<div id="path_stats_refsd">
+  <table class="list">
+    <tr><th>Domains</th><th>Hits</th></tr>
+    <%% initListClass
+       for row in @refs %>
+    <tr class="<%%=popListClass%>"><td><a href='<%%=row['referer']%>' title='<%%=row['resource']%>'><%%=row['domain']%></a></td><td><%%=row['total']%></td></tr>
+    <%% end %>
+  </table>
+</div>
+
+<h2>Search strings</h2>
+<div id="path_stats_refst">
+  <table class="list">
+    <tr><th>Search strings</th><th>Hits</th></tr>
+    <%% initListClass
+       for row in @searchterms %>
+    <tr class="<%%=popListClass%>"><td><%%=h row['searchterms']%></td><td><%%=row['count']%></td></tr>
+    <%% end %>
+  </table>
+</div>
+</div>
+</div>

data/templates/db/ip-to-country.mysql.sql

@@ -0,0 +1,22 @@
+drop table if exists iptocs;
+
+create table IF NOT EXISTS iptocs(
+ ip_from int(10) unsigned zerofill not null,
+ ip_to int(10) unsigned zerofill not null,
+ country_code2 char(2) not null,
+ country_code3 char(3) not null,
+ country_name varchar(50) not null,
+ unique index(ip_from, ip_to)
+);
+
+delete from iptocs;
+
+load data local infile 'ip-to-country.csv' 
+  into table iptocs
+  fields terminated by ',' enclosed by '"' 
+  lines terminated by '\r\n';
+
+alter table iptocs add column id bigint(20) NOT NULL auto_increment primary key;
+
+alter table iptocs add unique index (id);
+optimize table iptocs;

data/templates/db/railstat.mysql.sql

@@ -0,0 +1,43 @@
+CREATE TABLE rail_stats (
+  id int(11) unsigned NOT NULL auto_increment,
+  remote_ip varchar(15) default '',
+  country varchar(50) default '',
+  language VARCHAR(5) default '',
+  domain varchar(250) default '',
+  subdomain varchar(250) default '',
+  referer varchar(255) default '',
+  resource varchar(255) default '',
+  user_agent varchar(255) default '',
+  platform varchar(50) default '',
+  browser varchar(50) default '',
+  version varchar(15) default '',
+  dt int(10) unsigned default '0',
+  screen_size varchar(10) default null,
+  colors varchar(10) default null,
+  java varchar(10) default null,
+  java_enabled varchar(10) default null,
+  flash varchar(10) default null,
+  UNIQUE KEY id (id),
+  INDEX(subdomain)
+) TYPE=MyISAM;
+
+CREATE TABLE search_terms (
+  id int(11) unsigned NOT NULL auto_increment,
+  subdomain varchar(250) default '',
+  searchterms varchar(255) NOT NULL default '',
+  count int(10) unsigned NOT NULL default '0',
+  domain varchar(250),
+  PRIMARY KEY  (id),
+  INDEX(subdomain)
+) TYPE=MyISAM;
+
+CREATE TABLE iptocs (
+ id int(11) unsigned NOT NULL auto_increment,  
+ ip_from int(10) unsigned zerofill not null,
+ ip_to int(10) unsigned zerofill not null,
+ country_code2 char(2) not null,
+ country_code3 char(3) not null,
+ country_name varchar(50) not null,
+ PRIMARY KEY  (id),
+ unique index(ip_from, ip_to)
+);

data/templates/lib/path_tracker.rb

@@ -0,0 +1,325 @@
+require 'uri'
+require 'socket'
+
+module PathTracker
+  public 
+
+  def track_path
+    begin
+      referer = @params['referer'] # env['HTTP_REFERER']
+      req_uri = @params['doc'] # env['REQUEST_URI']
+      
+      req_uri = @request.env['HTTP_REFERER'] if req_uri.nil?
+      
+      req_uri = get_doc_url(req_uri)
+      
+      size = @params['size']
+      colors = @params['colors']
+      java = @params['java']
+      je = @params['je']
+      flash = @params['flash']
+              
+      env = @request.env.nil? ? {'HTTP_USER_AGENT' => nil, 'HTTP_REFERER' => nil,
+        'REMOTE_ADDR' => nil, 'HTTP_ACCEPT_LANGUAGE' => nil, 'REQUEST_URI' => nil} : @request.env
+      br = parse_user_agent(env['HTTP_USER_AGENT'])
+      subdomain = detect_subdomain
+      domain = get_urls_host(referer)
+      
+      sniff_keywords(domain, subdomain, referer);
+
+      @client_country = Iptoc.find_by_ip_address(@request.env['REMOTE_ADDR'])
+
+      RailStat.create("remote_ip" => env['REMOTE_ADDR'],
+                      "country" => ((@client_country.nil? or @client_country.country_name.nil?) ? '' : @client_country.country_name),
+                      "language" => determine_lang(env['HTTP_ACCEPT_LANGUAGE']),
+                      "domain" => domain,
+                      "subdomain" => subdomain,
+                      "referer" => referer,
+                      "resource" => req_uri,
+                      "user_agent" => env['HTTP_USER_AGENT'],
+                      "platform" => br['platform'],
+                      "browser" => br['browser'],
+                      "version" => br['version'],
+                      "dt" => Time.now,
+                      "screen_size" => size,
+                      "colors" => colors,
+                      "java" => java,
+                      "java_enabled" => je,
+                      "flash" => flash)
+    rescue Exception => e
+      RAILS_DEFAULT_LOGGER.error("Error on path track #{e.backtrace.join('\n')}" )
+    end                  
+    ""                
+  end
+    
+  private
+    
+  def determine_lang(language) 
+    ret = "empty"
+    unless language.nil? or language == ''
+      # Capture up to the first delimiter (, found in Safari)
+      begin
+        ret = /([^,;]*)/.match(language)[0];
+      rescue
+      end  
+    end
+    return ret
+  end
+    
+  def get_urls_host(ref)
+    if host = parse_url(ref)
+      if host == SITE_NAME
+        return nil
+      else
+        host
+      end
+    else
+      return nil
+    end  
+  end
+  
+  def parse_url(arg)
+    if arg.nil?
+      return nil
+    else
+      URI::split(arg.gsub(' ','%20'))[2] 
+    end
+  end
+  
+  def get_doc_url(arg)
+    if arg.nil?
+      return nil
+    else
+      URI::split(arg.gsub(' ','%20'))[5] 
+    end
+  end
+  
+  def get_params_url(arg)
+    if arg.nil?
+      return nil
+    else
+      URI::split(arg.gsub(' ','%20'))[7] 
+    end
+  end
+  
+  def parse_user_agent(ua)
+    browser = {'platform' => "unknown", 'browser' => "unknown",
+      'version' => "unknown", 'majorver' => "unknown", 'minorver' => "unknown"}
+    begin
+      # Test for platform
+      if ua =~ /Win/i
+        browser['platform'] = "Windows";
+      elsif ua =~ /Mac/i
+        browser['platform'] = "Macintosh";
+      elsif ua =~ /Linux/i
+        browser['platform'] = "Linux";
+      end
+      
+      
+      # Test for browser type
+      if ua =~ /Mozilla\/4/i && !ua =~ /compatible/i
+        browser['browser'] = "Netscape";
+        begin 
+          browser['version'] = /Mozilla\/([[:digit:]\.]+)/i.match(ua)[1] 
+        rescue 
+        end
+      end
+      if ua =~ /Mozilla\/5/i || ua =~ /Gecko/i
+        browser['browser'] = "Mozilla";
+        begin 
+          browser['version'] = /rv(:| )([[:digit:]\.]+)/i.match(ua)[2]  
+        rescue 
+        end
+      end
+      if ua =~ /Safari/i
+        browser['browser'] = "Safari";
+        browser['platform'] = "Macintosh";
+        begin 
+          browser['version'] = /Safari\/([[:digit:]\.]+)/i.match(ua)[1]  
+        rescue 
+        end
+        
+        if browser['version'] =~ /125/i
+          browser['version']   = 1.2;
+          browser['majorver']  = 1;
+          browser['minorver']  = 2;
+        elsif browser['version'] =~ /100/i
+          browser['version']   = 1.1;
+          browser['majorver']  = 1;
+          browser['minorver']  = 1;
+        elsif browser['version'] =~ /85/i
+          browser['version']   = 1.0;
+          browser['majorver']  = 1;
+          browser['minorver']  = 0;
+        else 
+          begin
+            if browser['version'].to_i < 85 
+              browser['version']   = "Pre-1.0 Beta";
+            end
+          rescue
+          end  
+        end
+      end
+      if ua =~ /iCab/i
+        browser['browser'] = "iCab";
+        begin 
+          browser['version'] = /iCab\/([[:digit:]\.]+)/i.match(ua)[1]  
+        rescue 
+        end
+      end
+      if ua =~ /Firefox/i
+        browser['browser'] = "Firefox";
+        begin 
+          browser['version'] = /Firefox\/([[:digit:]\.]+)/i.match(ua)[1]  
+        rescue 
+        end
+      end
+      if ua =~ /Firebird/i
+        browser['browser'] = "Firebird";
+        begin 
+          browser['version'] = /Firebird\/([[:digit:]\.]+)/i.match(ua)[1]  
+        rescue 
+        end
+      end
+      if ua =~ /Phoenix/i
+        browser['browser'] = "Phoenix";
+        begin 
+          browser['version'] = /Phoenix\/([[:digit:]\.]+)/i.match(ua)[1]  
+        rescue 
+        end
+      end
+      if ua =~ /Camino/i
+        browser['browser'] = "Camino";
+        begin 
+          browser['version'] = /Camino\/([[:digit:]\.]+)/i.match(ua)[1]  
+        rescue 
+        end
+      end
+      if ua =~ /Chimera/i
+        browser['browser'] = "Chimera";
+        begin 
+          browser['version'] = /Chimera\/([[:digit:]\.]+)/i.match(ua)[1] 
+        rescue 
+        end
+      end
+      if ua =~ /Netscape/i
+        browser['browser'] = "Netscape";
+        begin 
+          browser['version'] = /Netscape[0-9]?\/([[:digit:]\.]+)/i.match(ua)[1] 
+        rescue 
+        end
+      end
+      if ua =~ /MSIE/i
+        browser['browser'] = "IE";
+        begin 
+          browser['version'] = /MSIE ([[:digit:]\.]+)/i.match(ua)[1] 
+        rescue 
+        end
+      end
+      if ua =~ /Opera/i
+        browser['browser'] = "Opera";
+        begin 
+          browser['version'] = /Opera( |\/)([[:digit:]\.]+)/i.match(ua)[1] 
+        rescue 
+        end
+      end
+      if ua =~ /OmniWeb/i
+        browser['browser'] = "OmniWeb";
+        begin 
+          browser['version'] = /OmniWeb\/([[:digit:]\.]+)/i.match(ua)[1] 
+        rescue 
+        end
+      end
+      if ua =~ /Konqueror/i
+        browser['platform'] = "Linux";
+        browser['browser'] = "Konqueror";
+        begin 
+          browser['version'] = /Konqueror\/([[:digit:]\.]+)/i.match(ua)[1] 
+        rescue 
+        end
+      end
+      if ua =~ /Crawl/i || ua =~ /bot/i || ua =~ /slurp/i || ua =~ /spider/i
+        browser['browser'] = "Crawler/Search Engine";
+      end
+      if ua =~ /Lynx/i
+        browser['browser'] = "Lynx";
+        begin 
+          browser['version'] = /Lynx\/([[:digit:]\.]+)/i.match(ua)[1] 
+        rescue 
+        end
+      end
+      if ua =~ /Links/i
+        browser['browser'] = "Links";
+        begin 
+          browser['version'] = /\(([[:digit:]\.]+)/i.match(ua)[1] 
+        rescue 
+        end
+      end      
+      
+      # Determine browser versions
+      unless browser['browser'] == 'Safari' or browser['browser'] == "unknown" or browser['browser'] == "Crawler/Search Engine" or browser['version'] == "unknown"
+        # Make sure we have at least .0 for a minor version
+        browser['version'] = (!browser['version']=~/\./ ? "#{browser['version']}.0" : browser['version'])
+        
+        v = /^([0-9]*).(.*)$/.match(browser['version'])
+        begin 
+          browser['majorver'] = v[1];
+          browser['minorver'] = v[2];
+        rescue
+        end  
+      end
+      if browser['version'].nil? or browser['version'] == "" or browser['version'] == '.0'
+        browser['version']  = "unknown";
+        browser['majorver'] = "unknown";
+        browser['minorver'] = "unknown";
+      end
+    rescue Exception => e
+    end
+    return browser
+  end
+  
+  public 
+  
+  def sniff_keywords(domain, subdomain, referer) 
+    begin
+      searchterms = ''
+      uripars = get_params_url(referer) unless referer.nil?
+      params = CGI.parse(get_params_url(referer)) if uripars and not(uripars.nil?)
+      if domain =~ /google\./i
+        # Googles search terms are in "q"
+        searchterms = params['q']
+      elsif domain =~ /alltheweb\./i
+        # All the Web search terms are in "q"
+        searchterms = params['q']
+      elsif domain =~ /yahoo\./i
+        tempwords = referer.scan(/(\/K=)([^\/]*)(\/)/)
+        if tempwords and tempwords[0]
+          searchterms = tempwords[0][1]
+        else
+          searchterms = params['va']
+        end
+      elsif domain =~ /search\.aol\./i
+        # Yahoo search terms are in "query"
+        searchterms = params['query']
+      elsif domain =~ /search\.msn\./i
+        # MSN search terms are in "q"
+        searchterms = params['q']
+      end
+      SearchTerm.register(searchterms, domain, subdomain) unless searchterms.nil? or searchterms == ''
+    rescue Exception=>ex
+      RAILS_DEFAULT_LOGGER.debug("#{ex} - #{ex.backtrace.join("\n\t")}")
+    end
+  end
+  
+  private 
+  
+  def detect_subdomain
+    subd = ''
+    begin
+      subd = ((@request.subdomains and @request.subdomains.first) ? @request.subdomains.first : nil)
+    rescue Exception => e
+      logger.error("Error on subdomain parse #{e.backtrace.join('\n')}" )
+    end
+    return subd
+  end 
+end