birdwatcher 0.1.0

data/lib/birdwatcher/modules/users/klout_id.rb

@@ -0,0 +1,62 @@
+module Birdwatcher
+  module Modules
+    module Users
+      class KloutId < Birdwatcher::Module
+        self.meta = {
+          :name        => "User Klout IDs",
+          :description => "Enrich users with their Klout ID",
+          :author      => "Michael Henriksen <michenriksen@neomailbox.ch>",
+          :options     => {
+            "THREADS" => {
+              :value       => 5,
+              :description => "Number of concurrent threads",
+              :required    => false
+            }
+          }
+        }
+
+        def self.info
+<<-INFO
+The User Klout IDs module can populate the current workspace's users with their
+Klout ID. Having a Klout ID on users makes it possible to gather information
+with other modules such as #{'users/klout_influence'.bold}, #{'users/klout_topics'.bold} and
+#{'users/klout_score'.bold}.
+
+#{'Note:'.bold} Birdwatcher must be configured with one or more Klout API keys
+in order to work.
+INFO
+        end
+
+        def run
+          if !klout_client
+            error("Birdwatcher has not been configured with any Klout API keys")
+            return false
+          end
+          users = current_workspace.users_dataset.order(:screen_name)
+          if users.empty?
+            error("There are no users to process")
+            return false
+          end
+          threads = thread_pool(option_setting("THREADS").to_i)
+          users.each do |user|
+            threads.process do
+              begin
+                klout_id = klout_client.get_id(user.screen_name)
+                if klout_id.nil?
+                  warn("User #{user.screen_name.bold} doesn't have a Klout ID; skipping")
+                  next
+                end
+                user.klout_id = klout_id
+                user.save
+                info("User #{user.screen_name.bold} has a Klout ID: #{klout_id.to_s.bold}")
+              rescue => e
+                error("Processing of #{user.screen_name.bold} failed (#{e.class})")
+              end
+            end
+          end
+          threads.shutdown
+        end
+      end
+    end
+  end
+end

data/lib/birdwatcher/modules/users/klout_influence.rb

@@ -0,0 +1,83 @@
+module Birdwatcher
+  module Modules
+    module Users
+      class KloutInfluence < Birdwatcher::Module
+        self.meta = {
+          :name        => "User Klout Influence",
+          :description => "Enrich users with their Klout influence",
+          :author      => "Michael Henriksen <michenriksen@neomailbox.ch>",
+          :options     => {
+            "THREADS" => {
+              :value       => 5,
+              :description => "Number of concurrent threads",
+              :required    => false
+            }
+          }
+        }
+
+        def self.info
+<<-INFO
+The User Klout Influence module can be used to gather an influence graph of all
+users in the currently active workspace. The Klout Influence API can tell who
+users are being influenced by as well as who they are influencing.
+
+The influence graph can be generated with the #{'users/influence_graph'.bold} module
+when the raw data has been gathered with this module.
+
+#{'Note:'.bold} This module requires that users have been enriched with their
+Klout ID from the #{'users/klout_id'.bold} module. It also requires that Birdwatcher has
+been configured with one or more Klout API keys in order to work.
+INFO
+        end
+
+        def run
+          if !klout_client
+            error("Birdwatcher has not been configured with any Klout API keys")
+            return false
+          end
+          users = current_workspace.users_dataset.where("klout_id IS NOT NULL").order(:screen_name)
+          if users.empty?
+            error("There are no users with Klout IDs")
+            return false
+          end
+          threads          = thread_pool(option_setting("THREADS").to_i)
+          influencer_mutex = Mutex.new
+          influencee_mutex = Mutex.new
+          users.each do |user|
+            threads.process do
+              begin
+                if influence = klout_client.get_influence(user.klout_id)
+                  influence[:influencers].each do |screen_name|
+                    db_influencer = influencer_mutex.synchronize do
+                      Birdwatcher::Models::Influencer.find_or_create(:workspace_id => current_workspace.id, :screen_name => screen_name)
+                    end
+                    if !user.influencers.include?(db_influencer)
+                      user.add_influencer(db_influencer)
+                    end
+                  end
+                  influence[:influencees].each do |screen_name|
+                    db_influencee = influencee_mutex.synchronize do
+                      Birdwatcher::Models::Influencee.find_or_create(:workspace_id => current_workspace.id, :screen_name => screen_name)
+                    end
+                    if !user.influencees.include?(db_influencee)
+                      user.add_influencee(db_influencee)
+                    end
+                  end
+                  info("User #{user.screen_name.bold} is influenced by: #{influence[:influencers].map(&:bold).join(', ')}")
+                  info("User #{user.screen_name.bold} is influencing: #{influence[:influencees].map(&:bold).join(', ')}")
+                else
+                  error("Could not get Klout influence for #{user.screen_name.bold}")
+                end
+                user.save
+              rescue => e
+                error("Processing of #{user.screen_name.bold} failed (#{e.class})")
+                info(e.backtrace.join("\n"))
+              end
+            end
+          end
+          threads.shutdown
+        end
+      end
+    end
+  end
+end

data/lib/birdwatcher/modules/users/klout_score.rb

@@ -0,0 +1,64 @@
+module Birdwatcher
+  module Modules
+    module Users
+      class KloutScore < Birdwatcher::Module
+        self.meta = {
+          :name        => "User Klout Score",
+          :description => "Enrich users with their Klout score",
+          :author      => "Michael Henriksen <michenriksen@neomailbox.ch>",
+          :options     => {
+            "THREADS" => {
+              :value       => 5,
+              :description => "Number of concurrent threads",
+              :required    => false
+            }
+          }
+        }
+
+        def self.info
+<<-INFO
+The User Klout Score module can be used to retrieve the Klout Score of all users
+in the currently active workspace.
+
+The Klout score is a score between 1-100 and represents a user's influence. The
+more influential a user is, the higher their Klout score. Read more about how it
+works here: https://klout.com/corp/score
+
+#{'Note:'.bold} This module requires that users have been enriched with their
+Klout ID from the #{'users/klout_id'.bold} module. It also requires that Birdwatcher has
+been configured with one or more Klout API keys in order to work.
+INFO
+        end
+
+        def run
+          if !klout_client
+            error("Birdwatcher has not been configured with any Klout API keys")
+            return false
+          end
+          users = current_workspace.users_dataset.where("klout_id IS NOT NULL").order(:screen_name)
+          if users.empty?
+            error("There are no users with Klout IDs")
+            return false
+          end
+          threads = thread_pool(option_setting("THREADS").to_i)
+          users.each do |user|
+            threads.process do
+              begin
+                if klout_score = klout_client.get_score(user.klout_id)
+                  user.klout_score = klout_score
+                  info("User #{user.screen_name.bold} has a Klout score of #{klout_score.to_s.bold}")
+                else
+                  error("Could not get Klout score for #{user.screen_name.bold}")
+                end
+                user.save
+              rescue => e
+                error("Processing of #{user.screen_name.bold} failed (#{e.class})")
+              end
+            end
+          end
+          threads.shutdown
+        end
+      end
+    end
+  end
+end

data/lib/birdwatcher/modules/users/klout_topics.rb

@@ -0,0 +1,72 @@
+module Birdwatcher
+  module Modules
+    module Users
+      class KloutTopics < Birdwatcher::Module
+        self.meta = {
+          :name        => "User Klout Topics",
+          :description => "Enrich users with their Klout topics",
+          :author      => "Michael Henriksen <michenriksen@neomailbox.ch>",
+          :options     => {
+            "THREADS" => {
+              :value       => 5,
+              :description => "Number of concurrent threads",
+              :required    => false
+            }
+          }
+        }
+
+        def self.info
+<<-INFO
+The User Klout Topics module can be used to retrieve the general topics that
+users in the currently active workspace are tweeting about.
+
+#{'Note:'.bold} This module requires that users have been enriched with their
+Klout ID from the #{'users/klout_id'.bold} module. It also requires that Birdwatcher has
+been configured with one or more Klout API keys in order to work.
+INFO
+        end
+
+        def run
+          if !klout_client
+            error("Birdwatcher has not been configured with any Klout API keys")
+            return false
+          end
+          users = current_workspace.users_dataset.where("klout_id IS NOT NULL").order(:screen_name)
+          if users.empty?
+            error("There are no users with Klout IDs")
+            return false
+          end
+          threads = thread_pool(option_setting("THREADS").to_i)
+          mutex   = Mutex.new
+          users.each do |user|
+            threads.process do
+              begin
+                if klout_topics = klout_client.get_topics(user.klout_id)
+                  if klout_topics.empty?
+                    warn("User #{user.screen_name.bold} has no topics; skipping")
+                    next
+                  end
+                  klout_topics.each do |topic|
+                    db_topic = mutex.synchronize do
+                      Birdwatcher::Models::KloutTopic.find_or_create(:workspace_id => current_workspace.id, :topic => topic)
+                    end
+                    if !user.klout_topics.include?(db_topic)
+                      user.add_klout_topic(db_topic)
+                    end
+                  end
+                  info("User #{user.screen_name.bold} has topics: #{klout_topics.map(&:bold).join(', ')}")
+                else
+                  error("Could not get Klout topics for #{user.screen_name.bold}")
+                end
+                user.save
+              rescue => e
+                error("Processing of #{user.screen_name.bold} failed (#{e.class})")
+              end
+            end
+          end
+          threads.shutdown
+        end
+      end
+    end
+  end
+end

data/lib/birdwatcher/modules/users/social_graph.rb

@@ -0,0 +1,110 @@
+module Birdwatcher
+  module Modules
+    module Users
+      class SocialGraph < Birdwatcher::Module
+        self.meta = {
+          :name        => "Social Graph",
+          :description => "Graphs the social relations between users",
+          :author      => "Michael Henriksen <michenriksen@neomailbox.ch>",
+          :options     => {
+            "DEST" => {
+              :value       => nil,
+              :description => "Destination file",
+              :required    => true
+            },
+            "USERS" => {
+              :value       => nil,
+              :description => "Space-separated list of screen names (all users if empty)",
+              :required    => false
+            },
+            "MIN_WEIGHT" => {
+              :value       => 10,
+              :description => "Percentage of the highest edge weight to be considered minimum edge weight",
+              :required    => true
+            },
+            "FORMAT" => {
+              :value       => "png",
+              :description => "Destination file format (any format supported by Graphviz)",
+              :required    => true
+            }
+          }
+        }
+
+        def self.info
+<<-INFO
+The Social Graph module generates an undirected graph between users in the
+currently active workspace. The edges between users will be weighted by simply
+counting the amount of times they mutually mention each other in statuses. The
+module can identify social clusters within the collection of users.
+
+The generated graph is in PNG format.
+INFO
+        end
+
+        def run
+          if !GraphViz::Constants::FORMATS.include?(option_setting("FORMAT"))
+            error("Unsupported format: #{option_setting('FORMAT').bold}")
+            return false
+          end
+          if screen_names = option_setting("USERS")
+            users = current_workspace.users_dataset
+              .where("screen_name IN ?", screen_names.split(" ").map(&:strip))
+              .order(:screen_name)
+          else
+            users = current_workspace.users_dataset.order(:screen_name)
+          end
+          if users.empty?
+            error("There are no users to process")
+            return false
+          end
+          graph = GraphViz.new(:G, :type => :graph, :use => "sfdp", :overlap => "prism", :splines => "curved")
+          edge_weights = {}
+          highest_edge_weight = 0
+          nodes = {}
+          users.each do |user|
+            task("Calculating social graph for #{user.screen_name.bold}...") do
+              users.each do |other_user|
+                next if user.id == other_user.id
+                edge_weights[user.screen_name] ||= {}
+                edge_weights[other_user.screen_name] ||= {}
+                next if (edge_weights[user.screen_name].key?(other_user.screen_name) && edge_weights[other_user.screen_name].key?(user.screen_name))
+                edge_weights[user.screen_name][other_user.screen_name] = user.statuses_dataset.where("text LIKE ?", "%#{other_user.screen_name}%").count
+                edge_weights[other_user.screen_name][user.screen_name] = other_user.statuses_dataset.where("text LIKE ?", "%#{user.screen_name}%").count
+                total_edge_weight = edge_weights[user.screen_name][other_user.screen_name] + edge_weights[other_user.screen_name][user.screen_name]
+                highest_edge_weight = total_edge_weight if (total_edge_weight > highest_edge_weight)
+              end
+            end
+          end
+          task("Generating social graph...") do
+            edge_weights.each_pair do |user, user_graph|
+              user_graph.each_pair do |other_user, edge_weight|
+                total_edge_weight = edge_weight + edge_weights[other_user][user]
+                percentage = (total_edge_weight.to_f / highest_edge_weight.to_f * 100).to_i
+                if percentage >= option_setting("MIN_WEIGHT").to_i
+                  nodes[user] ||= graph.add_nodes(user)
+                  nodes[other_user] ||= graph.add_nodes(other_user)
+                  case percentage
+                  when 0..25
+                    pen_width = 1
+                  when 26..50
+                    pen_width = 2
+                  when 51..75
+                    pen_width = 4
+                  when 76..100
+                    pen_width = 5
+                  end
+                  graph.add_edges(nodes[user], nodes[other_user], :weight => total_edge_weight, :penwidth => pen_width, :color => "lightblue", :fontcolor => "cornflowerblue", :label => total_edge_weight.to_s)
+                  edge_weights[other_user].delete(user)
+                end
+              end
+            end
+          end
+          task("Outputting graph...") do
+            graph.output(option_setting("FORMAT") => option_setting("DEST"))
+          end
+          info("Graph written to #{option_setting('DEST').bold}")
+        end
+      end
+    end
+  end
+end

data/lib/birdwatcher/punchcard.rb

@@ -0,0 +1,183 @@
+# Code adapted from https://github.com/jashank/punchcard-plot
+
+module Birdwatcher
+  class Punchcard
+    LEFT_PADDING = 10
+    TOP_PADDING  = 10
+
+    WIDTH = 1100
+
+    DAYS  = %w(Sat Fri Thu Wed Tue Mon Sun)
+    HOURS = %w(12am 1 2 3 4 5 6 7 8 9 10 11 12pm 1 2 3 4 5 6 7 8 9 10 11)
+
+    FONT_FACE = "sans-serif"
+
+    def initialize(timestamps)
+      @timestamps = timestamps
+    end
+
+    def generate(destination)
+      @data_log  = Hash.new { |h, k| h[k] = Hash.new }
+      final_data = []
+      @timestamps.each do |timestamp|
+        day  = timestamp.strftime("%a")
+        hour = timestamp.strftime("%H").to_i
+        @data_log[day][hour] = (@data_log[day][hour] || 0) + 1
+      end
+      @data_log.each do |d, hour_pair|
+        hour_pair.each do |h, value|
+          glr = @data_log[d][h] * 1.0
+          glr /= max_value
+          glr *= max_range
+          glrb = get_weight(glr)
+          final_data.push([glrb, get_x_y_from_day_and_hour(d, h)])
+        end
+      end
+      final_data.each do |x|
+        draw_circle(x[1], x[0])
+      end
+      surface.write_to_png(destination)
+    end
+
+    private
+
+    def width
+      WIDTH
+    end
+
+    def height
+      (width / 2.75).round(0)
+    end
+
+    def distance
+      distance = Math.sqrt((width * height) / 270.5).round
+      if distance % 2 == 1
+        distance -= 1
+      end
+      distance
+    end
+
+    def max_range
+      (distance / 2)**2
+    end
+
+    def left
+      (width / 18) + LEFT_PADDING
+    end
+
+    def indicator_length
+      height / 20
+    end
+
+    def top
+      indicator_length + TOP_PADDING
+    end
+
+    def get_x_y_from_day_and_hour(day, hour)
+      y = top + (DAYS.index(day.to_s) + 1) * distance
+      x = left + (hour.to_i + 1) * distance
+      [x, y]
+    end
+
+    def get_weight(number)
+      return 0 if number.zero?
+      (1..(distance / 2)).to_a.each do |i|
+        if i * i <= number && number < (i + 1) * (i + 1)
+          return i
+        end
+      end
+
+      if number == max_range
+        return distance/2-1
+      end
+
+      nil
+    end
+
+    def all_values
+      @all_values = []
+      @data_log.each do |d, e|
+        e.each do |h, i|
+          @all_values << @data_log[d][h]
+        end
+      end
+      @all_values
+    end
+
+    def max_value
+      all_values.sort.last
+    end
+
+    def surface
+      @surface ||= Cairo::ImageSurface.new(Cairo::FORMAT_ARGB32, width, height)
+    end
+
+    def context
+      @context ||= Cairo::Context.new(surface).tap do |c|
+        c.line_width = 1
+        c.set_source_rgb(1, 1, 1)
+        c.rectangle(0, 0, width, height)
+        c.fill
+
+        # Set black
+        c.set_source_rgb(0, 0, 0)
+
+        # Draw X and Y axis
+        c.move_to(left, top)
+        c.rel_line_to(0, 8 * distance)
+        c.rel_line_to(25 * distance, 0)
+        c.stroke
+
+        # Draw indicators on X and Y axis
+        x, y = left, top
+        8.times do
+          c.move_to(x, y)
+          c.rel_line_to(-indicator_length, 0)
+          c.stroke
+          y += distance
+        end
+
+        x += distance
+        26.times do
+          c.move_to(x, y)
+          c.rel_line_to(0, indicator_length)
+          c.stroke
+          x += distance
+        end
+
+        # Select font
+        c.select_font_face(FONT_FACE, Cairo::FONT_SLANT_NORMAL, Cairo::FONT_WEIGHT_NORMAL)
+
+        # Set and appropiate font size
+        c.set_font_size(Math.sqrt( (width * height) / 3055.6))
+
+        # Draw days on Y axis
+        x, y = (left - 5), (top + distance)
+        DAYS.each do |day|
+          t_ext = c.text_extents(day.to_s)
+          c.move_to(x - indicator_length - t_ext.width, y + t_ext.height / 2)
+          c.show_text(day.to_s)
+          y += distance
+        end
+
+        # Draw hours on X axis
+        x, y = (left + distance), (top + (7 + 1) * distance + 5)
+        HOURS.each do |hour|
+          t_ext = c.text_extents(hour.to_s)
+          c.move_to(x - t_ext.width / 2 - t_ext.x_bearing, y + indicator_length + t_ext.height / 2)
+          c.show_text(hour.to_s)
+          x += distance
+        end
+      end
+    end
+
+    def draw_circle(position, weight)
+      x, y  = position
+      alpha = (weight.to_f / max_value.to_f)
+      context.set_source_rgba(0, 0, 0, alpha)
+      context.move_to(x, y)
+      context.arc(x, y, weight, 0, 2 * Math::PI)
+      context.fill
+    end
+  end
+end

data/lib/birdwatcher/util.rb

@@ -0,0 +1,83 @@
+module Birdwatcher
+  module Util
+    def self.time_ago_in_words(time)
+      return "a very very long time ago" if time.year < 1800
+      secs = Time.now - time
+      return "just now" if secs > -1 && secs < 1
+      return "" if secs <= -1
+      pair = ago_in_words_pair(secs)
+      ary = ago_in_words_singularize(pair)
+      ary.size == 0 ? "" : ary.join(" and ") << " ago"
+    end
+
+    def self.ago_in_words_pair(secs)
+      [[60, :seconds], [60, :minutes], [24, :hours], [100_000, :days]].map{ |count, name|
+        if secs > 0
+          secs, n = secs.divmod(count)
+          "#{n.to_i} #{name}"
+        end
+      }.compact.reverse[0..1]
+    end
+
+    def self.ago_in_words_singularize(pair)
+      if pair.size == 1
+        pair.map! {|part| part[0, 2].to_i == 1 ? part.chomp("s") : part }
+      else
+        pair.map! {|part| part[0, 2].to_i == 1 ? part.chomp("s") : part[0, 2].to_i == 0 ? nil : part }
+      end
+      pair.compact
+    end
+
+    def self.parse_time(time)
+      ::Chronic.parse(time)
+    end
+
+    def self.strip_html(string)
+      string.to_s.gsub(/<\/?[^>]*>/, "")
+    end
+
+    def self.strip_control_characters(string)
+      string = string.to_s.uncolorize
+      string.split("").delete_if do |char|
+        char.ascii_only? and (char.ord < 32 or char.ord == 127)
+      end.join("")
+    end
+
+    def self.escape_html(string)
+      CGI.escapeHTML(string.to_s)
+    end
+
+    def self.unescape_html(string)
+      CGI.unescapeHTML(string.to_s)
+    end
+
+    def self.pluralize(count, singular, plural)
+      count == 1 ? "1 #{singular}" : "#{count} #{plural}"
+    end
+
+    def self.excerpt(text, max_length, omission = "...")
+      text = text.gsub(/\s/, " ").split(" ").map(&:strip).join(" ")
+      return text if text.length < max_length
+      text[0..max_length] + omission
+    end
+
+    def self.suppress_output(&block)
+      original_stdout = $stdout
+      $stdout = fake = StringIO.new
+      begin
+        yield
+      ensure
+        $stdout = original_stdout
+      end
+      fake.string
+    end
+
+    def self.suppress_warnings(&block)
+      warn_level = $VERBOSE
+      $VERBOSE = nil
+      result = block.call
+      $VERBOSE = warn_level
+      result
+    end
+  end
+end

data/lib/birdwatcher/version.rb

@@ -0,0 +1,3 @@
+module Birdwatcher
+  VERSION = "0.1.0"
+end