charted 0.0.8 → 0.1.0

data/lib/charted/app.rb

@@ -0,0 +1,59 @@
+module Charted
+  class App < Sinatra::Base
+    set :logging, true
+    set :raise_errors, false
+    set :show_exceptions, false
+
+    before do
+      @site = Site.first(domain: request.host)
+      halt(404) if @site.nil?
+      @visitor = @site.visitor_with_cookie(request.cookies['charted'])
+    end
+
+    get '/' do
+      if @visitor.nil?
+        @visitor = @site.add_visitor(
+          resolution: params[:resolution],
+          user_agent: request.user_agent,
+          ip_address: request.ip,
+          bucket: params[:bucket])
+        response.set_cookie(
+          'charted',
+          value: @visitor.cookie,
+          path: '/',
+          expires: (Date.today + 365*2).to_time)
+      end
+
+      begin
+        referrer = params[:referrer].to_s
+        referrer = nil if URI.parse(referrer).host == @site.domain || referrer =~ /^\s*$/
+      rescue URI::InvalidURIError
+        referrer = nil
+      end
+      @visitor.add_visit(
+        path: params[:path],
+        title: params[:title],
+        referrer: referrer)
+      @visitor.start_conversions(params[:conversions])
+      @visitor.start_experiments(params[:experiments])
+      '/**/'
+    end
+
+    get '/record' do
+      halt(404) if @visitor.nil?
+      @visitor.make_events(params[:events])
+      @visitor.end_goals(params[:goals])
+      '/**/'
+    end
+
+    error do
+      err = request.env['sinatra.error']
+      Pony.mail(
+        to: Charted.config.error_email,
+        subject: "[Charted Error] #{err.message}",
+        body: [request.env.to_s, err.message, err.backtrace].compact.flatten.join("\n")
+      ) if Charted.config.error_email
+      raise err
+    end
+  end
+end

data/lib/charted/command.rb

@@ -0,0 +1,210 @@
+module Charted
+  class Command
+    attr_accessor :config_loaded, :output
+    attr_reader :site
+
+    def clean(label=nil)
+      load_config
+      sys_exit("Please set 'delete_after' config.") if Charted.config.delete_after.nil?
+
+      threshold = Date.today - Charted.config.delete_after
+      Visit.where { created_at < threshold }.delete
+      Event.where { created_at < threshold }.delete
+      Conversion.where { created_at < threshold }.delete
+      Experiment.where { created_at < threshold }.delete
+      Visitor.where { created_at < threshold }.each do |visitor|
+        visitor.delete if visitor.visits.count == 0 &&
+          visitor.events.count == 0 &&
+          visitor.conversions.count == 0 &&
+          visitor.experiments.count == 0
+      end
+
+      if label
+        Event.where(label: label).delete
+        Conversion.where(label: label).delete
+        Experiment.where(label: label).delete
+      end
+    end
+
+    def dashboard
+      site_required
+      nodes = []
+      max_width = [`tput cols`.to_i / 2, 60].min
+      table = Dashes::Table.new.
+        align(:right, :right, :left).
+        row('Total', 'Unique', 'Visits').
+        separator
+      (0..11).each do |delta|
+        date = Charted.prev_month(Date.today, delta)
+        query = Charted::Visit.
+          join(:visitors, id: :visitor_id).
+          where(visitors__site_id: @site.id).
+          where('visits.created_at >= ? AND visits.created_at < ?', date, Charted.next_month(date))
+        visits = query.count
+        unique = query.select(:visitor_id).distinct.count
+        table.row(format(visits), format(unique), date.strftime('%B %Y'))
+      end
+      nodes += [table]
+      [[:browser, 'Browsers', Charted::Visitor],
+       [:resolution, 'Resolutions', Charted::Visitor],
+       [:platform, 'Platforms', Charted::Visitor],
+       [:country, 'Countries', Charted::Visitor],
+       [:title, 'Pages', Charted::Visit],
+       [:referrer, 'Referrers', Charted::Visit],
+       [:search_terms, 'Searches', Charted::Visit]].each do |field, column, type|
+        table = Dashes::Table.new.
+          max_width(max_width).
+          spacing(:min, :min, :max).
+          align(:right, :right, :left).
+          row('Total', '%', column).separator
+        rows = []
+        query = type.exclude(field => nil)
+        query = query.join(:visitors, id: :visitor_id) if type == Charted::Visit
+        query = query.where(visitors__site_id: @site.id)
+        total = query.count
+        query.group_and_count(field).each do |row|
+          count = row[:count]
+          label = row[:label].to_s.strip
+          next if label == ""
+          label = "#{label[0..37]}..." if label.length > 40
+          rows << [format(count), "#{((count / total.to_f) * 100).round}%", label]
+        end
+        add_truncated(table, rows)
+        nodes << table
+      end
+      table = Dashes::Table.new.
+        max_width(max_width).
+        spacing(:min, :min, :max).
+        align(:right, :right, :left).
+        row('Total', 'Unique', 'Events').
+        separator
+      rows = []
+      events = Charted::Event.join(:visitors, id: :visitor_id).where(visitors__site_id: @site.id)
+      events.group_and_count(:label).all.each do |row|
+        label, count = row[:label], row[:count]
+        unique = Charted::Visitor.
+          join(:events, visitor_id: :id).
+          where(site_id: @site.id, events__label: label).
+          select(:visitors__id).distinct.count
+        rows << [format(count), format(unique), label]
+      end
+      add_truncated(table, rows)
+      nodes << table
+
+      table = Dashes::Table.new.
+        max_width(max_width).
+        spacing(:min, :min, :max).
+        align(:right, :right, :left).
+        row('Start', 'End', 'Conversions').
+        separator
+      rows = []
+      conversions = Charted::Conversion.join(:visitors, id: :visitor_id).where(visitors__site_id: @site.id)
+      conversions.group_and_count(:label).all.each do |row|
+        label, count = row[:label], row[:count]
+        ended = Charted::Visitor.
+          join(:conversions, visitor_id: :id).
+          where(site_id: @site.id, conversions__label: label).
+          exclude(conversions__ended_at: nil).
+          select(:visitors__id).distinct.count
+        rows << [format(count), format(ended), label]
+      end
+      add_truncated(table, rows)
+      nodes << table
+
+      table = Dashes::Table.new.
+        max_width(max_width).
+        spacing(:min, :min, :max).
+        align(:right, :right, :left).
+        row('Start', 'End', 'Experiments').
+        separator
+      rows = []
+      experiments = Charted::Experiment.join(:visitors, id: :visitor_id).where(visitors__site_id: @site.id)
+      experiments.group_and_count(:label, :experiments__bucket).all.each do |row|
+        label, bucket, count = row[:label], row[:bucket], row[:count]
+        ended = Charted::Visitor.
+          join(:experiments, visitor_id: :id).
+          where(site_id: @site.id, experiments__label: label, experiments__bucket: bucket).
+          exclude(experiments__ended_at: nil).
+          select(:visitors__id).distinct.count
+        rows << [format(count), format(ended), "#{label}: #{bucket}"]
+      end
+      add_truncated(table, rows)
+      nodes << table
+
+      nodes.reject! do |node|
+        minimum = node.is_a?(Dashes::Table) ? 1 : 0
+        node.instance_variable_get(:@rows).size == minimum # TODO: hacked
+      end
+      print(Dashes::Grid.new.width(`tput cols`.to_i).add(*nodes))
+    end
+
+    def js
+      print(File.read(JS_FILE))
+    end
+
+    def migrate
+      load_config
+      Charted::Migrate.run
+      Charted.config.sites.each do |domain|
+        if Site.first(domain: domain).nil?
+          Site.create(domain: domain)
+        end
+      end
+    end
+
+    def site=(domain)
+      load_config
+      sites = Site.where(Sequel.like(:domain, "%#{domain}%")).all
+
+      if sites.length > 1
+        sys_exit("\"#{domain}\" ambiguous: #{sites.map(&:domain).join(', ')}")
+      elsif sites.length < 1
+        sys_exit("No sites matching \"#{domain}\"")
+      else
+        @site = sites.first
+      end
+    end
+
+    private
+    def load_config
+      return if @config_loaded
+      file = ENV['CHARTED_CONFIG']
+      sys_exit("Please set CHARTED_CONFIG to `config.ru` file.") if !File.exist?(file.to_s)
+      load(file)
+      @config_loaded = true
+    end
+
+    def sys_exit(reason)
+      print(reason)
+      ENV['RACK_ENV'] == 'test' ? raise(ExitError.new(reason)) : exit
+    end
+
+    def print(string)
+      ENV['RACK_ENV'] == 'test' ?  (@output ||= []) << string : puts(string)
+    end
+
+    def site_required
+      load_config
+      if @site.nil? && Site.count == 1
+        @site = Site.first
+      elsif @site.nil?
+        sys_exit('Please specify website with --site')
+      end
+    end
+
+    def format(num)
+      num.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse
+    end
+
+    def add_truncated(table, rows)
+      rows = rows.sort_by { |r| r[0].gsub(/[^\d]/, '').to_i }.reverse
+      if rows.length > 12 
+        rows = rows[0..11]
+        rows << ['...', '...', '...']
+      end
+      rows.each { |row| table.row(*row) }
+    end
+  end
+
+  class ExitError < RuntimeError; end
+end

data/lib/charted/config.rb

@@ -0,0 +1,30 @@
+module Charted
+  class << self
+    attr_accessor :database
+
+    def configure
+      yield self.config
+      Pony.options = config.email_options if !config.email_options.nil?
+      Charted.database = Sequel.connect(config.db_options)
+      require_relative 'model'
+    end
+
+    def config
+      @config ||= Config.new
+    end
+  end
+
+  class Config
+    def self.attr_option(*names)
+      names.each do |name|
+        define_method(name) do |*args|
+          value = args[0]
+          instance_variable_set("@#{name}".to_sym, value) if !value.nil?
+          instance_variable_get("@#{name}".to_sym)
+        end
+      end
+    end
+
+    attr_option :error_email, :email_options, :db_options, :delete_after, :sites
+  end
+end

data/lib/charted/model.rb

@@ -0,0 +1,161 @@
+module Charted
+  class Migrate
+    def self.run
+      Sequel.extension :migration
+      Sequel::Migrator.run(Charted.database, File.join(File.dirname(__FILE__), '..', '..', 'migrate'))
+      # re-parse the schema after table changes
+      [Site, Visitor, Visit, Event, Conversion, Experiment].each do |table|
+        table.dataset = table.dataset
+      end
+    end
+  end
+
+  module HasVisitor
+    def site
+      self.visitor.site
+    end
+  end
+
+  module Endable
+    def ended?
+      !!ended_at
+    end
+
+    def end!
+      self.ended_at = DateTime.now
+      self.save
+    end
+  end
+
+  class Site < Sequel::Model
+    one_to_many :visitors
+
+    def initialize(*args)
+      super
+      self.created_at ||= DateTime.now
+    end
+
+    def visitor_with_cookie(cookie)
+      visitor = self.visitors_dataset[cookie.to_s.split('-').first.to_i]
+      visitor && visitor.cookie == cookie ? visitor : nil
+    end
+  end
+
+  class Visitor < Sequel::Model
+    many_to_one :site
+    one_to_many :visits
+    one_to_many :events
+    one_to_many :conversions
+    one_to_many :experiments
+
+    def initialize(*args)
+      super
+      self.created_at ||= DateTime.now
+      self.bucket ||= rand(10)
+      self.secret = SecureRandom.hex(3)
+    end
+
+    def cookie
+      # TODO: raise if nil id, bucket, or secret
+      "#{self.id}-#{self.bucket}-#{self.secret}"
+    end
+
+    def user_agent=(user_agent)
+      ua = UserAgent.parse(user_agent)
+      self.browser = ua.browser
+      self.browser_version = ua.version
+      self.platform = ua.platform == 'X11' ? 'Linux' : ua.platform
+    end
+
+    def ip_address=(ip)
+      return if ip.to_s =~ /^\s*$/ || ip == '127.0.0.1'
+      name = GEOIP.country(ip).country_name
+
+      return if name =~ /^\s*$/ || name == 'N/A'
+      self.country = name
+    rescue SocketError
+      # invalid IP address, skip setting country
+    end
+
+    def make_events(labels)
+      labels.to_s.split(';').map(&:strip).map do |label|
+        add_event(label: label)
+      end
+    end
+
+    def start_conversions(labels)
+      labels.to_s.split(';').map(&:strip).map do |label|
+        conversions_dataset.first(label: label) || self.add_conversion(label: label)
+      end
+    end
+
+    def start_experiments(labels) # label:bucket;...
+      labels.to_s.split(';').map do |str|
+        label, bucket = str.split(':', 2).map(&:strip)
+        exp = experiments_dataset.first(label: label)
+        if exp
+          exp.update(bucket: bucket) if exp.bucket != bucket
+          exp
+        else
+          self.add_experiment(label: label, bucket: bucket)
+        end
+      end
+    end
+
+    def end_goals(labels)
+      labels.to_s.split(';').map(&:strip).each do |label|
+        exp = experiments_dataset.first(label: label)
+        exp.end! if exp
+        conv = conversions_dataset.first(label: label)
+        conv.end! if conv
+      end
+    end
+  end
+
+  class Visit < Sequel::Model
+    include HasVisitor
+    many_to_one :visitor
+
+    def initialize(*args)
+      super
+      self.created_at ||= DateTime.now
+    end
+
+    def before_save
+      self.search_terms = URI.parse(self.referrer).search_string if self.referrer.to_s !~ /^\s*$/
+      super
+    end
+  end
+
+  class Event < Sequel::Model
+    include HasVisitor
+    many_to_one :visitor
+
+    def initialize(*args)
+      super
+      self.created_at ||= DateTime.now
+    end
+  end
+
+  class Conversion < Sequel::Model
+    include HasVisitor
+    include Endable
+    many_to_one :visitor
+
+    def initialize(*args)
+      super
+      self.created_at ||= DateTime.now
+    end
+  end
+
+  class Experiment < Sequel::Model
+    include HasVisitor
+    include Endable
+    many_to_one :visitor
+
+    def initialize(*args)
+      super
+      self.created_at ||= DateTime.now
+    end
+  end
+end