birdwatcher 0.1.0

data/lib/birdwatcher/modules/statuses/word_cloud.rb

@@ -0,0 +1,205 @@
+module Birdwatcher
+  module Modules
+    module Statuses
+      class WordCloud < Birdwatcher::Module
+        self.meta = {
+          :name        => "Word Cloud",
+          :description => "Generates a word cloud from statuses",
+          :author      => "Michael Henriksen <michenriksen@neomailbox.ch>",
+          :options     => {
+            "DEST" => {
+              :value       => nil,
+              :description => "Destination file",
+              :required    => true
+            },
+            "USERS" => {
+              :value       => nil,
+              :description => "Space-separated list of screen names (all users if empty)",
+              :required    => false
+            },
+            "SINCE" => {
+              :value       => nil,
+              :description => "Process statuses posted since specified time (last 7 days if empty)",
+              :required    => false
+            },
+            "BEFORE" => {
+              :value       => nil,
+              :description => "Process statuses posted before specified time (from now if empty)",
+              :required    => false
+            },
+            "MIN_WORD_COUNT" => {
+              :value       => 3,
+              :description => "Exclude words mentioned fewer times than specified",
+              :required    => false
+            },
+            "MIN_WORD_LENGTH" => {
+              :value       => 3,
+              :description => "Exclude words smaller than specified",
+              :required    => false
+            },
+            "EXCLUDE_STOPWORDS" => {
+              :value       => true,
+              :description => "Exclude english stopwords",
+              :required    => false,
+              :boolean     => true
+            },
+            "EXCLUDE_COMMON" => {
+              :value       => true,
+              :description => "Exclude common english words",
+              :required    => false,
+              :boolean     => true
+            },
+            "EXCLUDE_WORDS" => {
+              :value       => nil,
+              :description => "Space-separated list of words to exclude",
+              :required    => false
+            },
+            "EXCLUDE_HASHTAGS" => {
+              :value       => false,
+              :description => "Exclude Hashtags",
+              :required    => false,
+              :boolean     => true
+            },
+            "EXCLUDE_MENTIONS" => {
+              :value       => true,
+              :description => "Exclude @username mentions",
+              :required    => false,
+              :boolean     => true
+            },
+            "INCLUDE_PAGE_TITLES" => {
+              :value       => false,
+              :description => "Include web page titles from shared URLs (requires crawling with urls/crawl)",
+              :required    => false,
+              :boolean     => true
+            },
+            "WORD_CAP" => {
+              :value       => 200,
+              :description => "Cap list of words to specified amount",
+              :required    => false
+            },
+            "PALETTE" => {
+              :value       => "#8F99AB #A3ADC2 #272A2F #474C55 #3D4148 #021121 #293642 #516982 #516982 #415569",
+              :description => "Space-separated list of hex color codes to use for word cloud",
+              :required    => true
+            },
+            "IMAGE_WIDTH" => {
+              :value       => 1024,
+              :description => "Image width in pixels",
+              :required    => true
+            },
+            "IMAGE_HEIGHT" => {
+              :value       => 1024,
+              :description => "Image height in pixels",
+              :required    => true
+            },
+          }
+        }
+
+        DEFAULT_EXCLUDED_WORDS = %w(rt via oh)
+
+        def self.info
+<<-INFO
+The Word Cloud module can generate a classic weighted word cloud from words used
+in statuses across all or specific users and between different times.
+
+The module is heavily configurable; have a look at the options with #{'show options'.bold}
+
+Please note that configuring the module with a long timespan might result in a
+very long execution time when the word cloud image is generated.
+
+The generated image will be in PNG format.
+INFO
+        end
+
+        def run
+          if option_setting("USERS")
+            user_ids = current_workspace.users_dataset.where("screen_name IN ?", option_setting("USERS").split(" ").map(&:strip)).map(&:id)
+            statuses = current_workspace.statuses_dataset.where("user_id IN ?", user_ids)
+          else
+            statuses = current_workspace.statuses_dataset
+          end
+          if option_setting("SINCE")
+            since = parse_time(option_setting("SINCE")).strftime("%Y-%m-%d")
+          else
+            since = (Date.today - 7).strftime("%Y-%m-%d")
+          end
+          if option_setting("BEFORE")
+            before = parse_time(option_setting("BEFORE")).strftime("%Y-%m-%d")
+          else
+            before = Time.now.strftime("%Y-%m-%d")
+          end
+          statuses = statuses.where("DATE(posted_at) >= DATE(?) AND DATE(posted_at) <= DATE(?)", since, before).all
+          if statuses.count.zero?
+            error("There are no statuses to process")
+            return false
+          end
+          prepare_exclusion_list
+          words        = {}
+          sorted_words = []
+          task("Processing #{statuses.count.to_s.bold} statuses...") do
+            statuses.each do |status|
+              split_into_words(status.text).each do |word|
+                next if exclude_word?(word)
+                words.key?(word) ? words[word] += 1 : words[word] = 1
+              end
+              if option_setting("INCLUDE_PAGE_TITLES")
+                status.urls_dataset
+                .where("title IS NOT NULL")
+                .where("final_url NOT LIKE 'https://twitter.com/%'")
+                .map(&:title).each do |page_title|
+                  split_into_words(page_title).each do |word|
+                    next if exclude_word?(word)
+                    words.key?(word) ? words[word] += 1 : words[word] = 1
+                  end
+                end
+              end
+            end
+            if option_setting("MIN_WORD_COUNT")
+              words.delete_if { |word, count| count < option_setting("MIN_WORD_COUNT").to_i }
+            end
+            sorted_words = words.sort_by { |word, count| count}.reverse
+            if option_setting("WORD_CAP")
+              sorted_words = sorted_words.take(option_setting("WORD_CAP").to_i)
+            end
+          end
+          task("Generating word cloud, patience please...") do
+            cloud = MagicCloud::Cloud.new(sorted_words,
+              :rotate  => :none,
+              :palette => option_setting("PALETTE").split(" ").map(&:strip)
+            ).draw(option_setting("IMAGE_WIDTH").to_i, option_setting("IMAGE_HEIGHT").to_i).to_blob { self.format = "png" }
+            File.open(option_setting("DEST"), "wb") { |f| f.write(cloud) }
+          end
+          info("Word cloud written to #{option_setting('DEST').bold}")
+        end
+
+        private
+
+        def prepare_exclusion_list
+          @exclusion_list = DEFAULT_EXCLUDED_WORDS
+          if option_setting("EXCLUDE_WORDS")
+            @exclusion_list += option_setting("EXCLUDE_WORDS").split(" ").map { |w| w.strip.downcase }
+          end
+          if option_setting("EXCLUDE_STOPWORDS")
+            @exclusion_list += read_data_file("english_stopwords.txt").split("\n").map { |w| w.strip.downcase }
+          end
+          if option_setting("EXCLUDE_COMMON")
+            @exclusion_list += read_data_file("top100Kenglishwords.txt").split("\n").map(&:strip)
+          end
+        end
+
+        def exclude_word?(word)
+          return true if word.empty?
+          return true if option_setting("MIN_WORD_LENGTH") && word.length < option_setting("MIN_WORD_LENGTH").to_i
+          return true if option_setting("EXCLUDE_HASHTAGS") && word.start_with?("#")
+          return true if option_setting("EXCLUDE_MENTIONS") && word.start_with?("@")
+          return true if @exclusion_list.include?(word)
+        end
+
+        def split_into_words(text)
+          text = text.downcase.strip.gsub(/https?:\/\/[\S]+/, "").gsub(/[^0-9a-z@#_ ]/i, " ")
+          text.split(" ").map(&:strip)
+        end
+      end
+    end
+  end
+end

data/lib/birdwatcher/modules/urls/crawl.rb

@@ -0,0 +1,138 @@
+module Birdwatcher
+  module Modules
+    module Urls
+      class Crawl < Birdwatcher::Module
+        self.meta = {
+          :name        => "URL Crawler",
+          :description => "Enrich gathered URLs with HTTP status codes, content types and page titles",
+          :author      => "Michael Henriksen <michenriksen@neomailbox.ch>",
+          :options     => {
+            "USER_AGENT" => {
+              :value       => nil,
+              :description => "Specific HTTP User-Agent to use (randomized user-agents if not set)",
+              :required    => false
+            },
+            "TIMEOUT" => {
+              :value       => Birdwatcher::HttpClient::DEFAULT_TIMEOUT,
+              :description => "Request timeout in seconds",
+              :required    => false
+            },
+            "RETRIES" => {
+              :value       => Birdwatcher::HttpClient::DEFAULT_RETRIES,
+              :description => "Amount of retries on failed requests",
+              :required    => false
+            },
+            "RETRY_FAILED" => {
+              :value       => false,
+              :description => "Retry previously failed crawls",
+              :required    => false,
+              :boolean     => true
+            },
+            "PROXY_ADDR" => {
+              :value       => nil,
+              :description => "HTTP proxy address to use for requests",
+              :required    => false
+            },
+            "PROXY_PORT" => {
+              :value       => nil,
+              :description => "HTTP proxy port to use for requests",
+              :required    => false
+            },
+            "PROXY_USER" => {
+              :value       => nil,
+              :description => "HTTP proxy user to use for requests",
+              :required    => false
+            },
+            "PROXY_PASS" => {
+              :value       => nil,
+              :description => "HTTP proxy user to use for requests",
+              :required    => false
+            },
+            "THREADS" => {
+              :value       => 10,
+              :description => "The number of concurrent threads",
+              :required    => false
+            }
+          }
+        }
+
+        PAGE_TITLE_REGEX = /<title>(.*?)<\/title>/i
+
+        def self.info
+<<-INFO
+The URL Crawler module crawls shared URLs and enriches them with additional
+information:
+
+  * HTTP status code (200, 404, 500, etc.)
+  * Content type (application/html, application/pdf, etc)
+  * Page title (if HTML document)
+
+Page titles can be included in the Word Cloud generated with the
+#{'statuses/word_cloud'.bold} module.
+
+#{'CAUTION:'.bold} Depending on the users in the workspace, it might not be safe
+to blindly request shared URLs. Consider using the #{'PROXY_ADDR'.bold} and #{'PROXY_PORT'.bold}
+module options.
+INFO
+        end
+
+        def run
+          if option_setting("RETRY_FAILED")
+            urls = current_workspace.urls_dataset
+              .where("crawled_at IS NULL or (crawled_at IS NOT NULL AND http_status IS NULL)")
+              .order(Sequel.desc(:created_at))
+          else
+            urls = current_workspace.urls_dataset
+              .where(:crawled_at => nil)
+              .order(Sequel.desc(:created_at))
+          end
+          if urls.empty?
+            error("There are currently no URLs in this workspace")
+            return false
+          end
+          threads     = thread_pool(option_setting("THREADS").to_i)
+          http_client = Birdwatcher::HttpClient.new(
+            :timeout        => option_setting("TIMEOUT").to_i,
+            :retries        => option_setting("RETRIES").to_i,
+            :user_agent     => option_setting("USER_AGENT"),
+            :http_proxyaddr => option_setting("PROXY_ADDR"),
+            :http_proxyport => (option_setting("PROXY_PORT") ? option_setting("PROXY_PORT").to_i : nil),
+            :http_proxyuser => option_setting("PROXY_USER"),
+            :http_proxypass => option_setting("PROXY_PASS")
+          )
+          urls.each do |url|
+            threads.process do
+              begin
+                Timeout::timeout(option_setting("TIMEOUT").to_i * 2) do
+                  response = http_client.do_head(url.url)
+                  url.final_url    = response.url
+                  url.http_status  = response.status
+                  url.content_type = response.headers["content-type"]
+                  if response.headers.key?("content-type") && response.headers["content-type"].include?("text/html")
+                    url.title = extract_page_title(http_client.do_get(response.url).body)
+                  end
+                  url.crawled_at = Time.now
+                  url.save
+                  info("Crawled #{url.url.bold} (#{response.status} - #{response.headers["content-type"]})")
+                end
+              rescue => e
+                url.crawled_at = Time.now
+                url.save
+                error("Crawling failed for #{url.url.bold} (#{e.class})")
+              end
+            end
+          end
+          threads.shutdown
+        end
+
+        private
+
+        def extract_page_title(body)
+          title = body.scan(PAGE_TITLE_REGEX).first
+          return nil if title.nil?
+          CGI.unescapeHTML(title.first)
+        end
+      end
+    end
+  end
+end

data/lib/birdwatcher/modules/urls/most_shared.rb

@@ -0,0 +1,98 @@
+module Birdwatcher
+  module Modules
+    module Urls
+      class MostShared < Birdwatcher::Module
+        self.meta = {
+          :name        => "Most Shared URLs",
+          :description => "Lists shared URLs ordered from most to least shared",
+          :author      => "Michael Henriksen <michenriksen@neomailbox.ch>",
+          :options     => {
+            "USERS" => {
+              :value       => nil,
+              :description => "Space-separated list of screen names (all users if empty)",
+              :required    => false
+            },
+            "MIN_SHARE_COUNT" => {
+              :value       => 2,
+              :description => "Exclude URLS shared fewer times than specified",
+              :required    => false
+            },
+            "SINCE" => {
+              :value       => nil,
+              :description => "List URLs shared since specified time (last 7 days if empty)",
+              :required    => false
+            },
+            "BEFORE" => {
+              :value       => nil,
+              :description => "List URLs shared before specified time (from now if empty)",
+              :required    => false
+            }
+          }
+        }
+
+         def self.info
+<<-INFO
+The Most Shared URLs module can show a simple list of shared URLs ordered from
+most to least shared. If a URL has been shared by several people, it is a good
+indication that it has important or interesting information.
+
+To enhance the functionality of this module, it is recommended to run the
+#{'urls/crawl'.bold} module first in order to get information on the URLs such
+as HTTP status codes, content types and page titles. If the information is
+available, this module will display it.
+INFO
+        end
+
+        def run
+          if option_setting("SINCE")
+            since = parse_time(option_setting("SINCE")).strftime("%Y-%m-%d")
+          else
+            since = (Date.today - 7).strftime("%Y-%m-%d")
+          end
+          if option_setting("BEFORE")
+            before = parse_time(option_setting("BEFORE")).strftime("%Y-%m-%d")
+          else
+            before = Time.now.strftime("%Y-%m-%d")
+          end
+          if option_setting("USERS")
+            user_ids = current_workspace.users_dataset.where("screen_name IN ?", option_setting("USERS").split(" ").map(&:strip)).map(&:id)
+            urls = database["SELECT urls.url, urls.final_url, urls.title, urls.http_status, urls.content_type, count(statuses_urls.*) AS count
+              FROM urls
+              INNER JOIN statuses_urls
+                ON statuses_urls.url_id = urls.id
+              INNER JOIN statuses
+                ON statuses_urls.status_id = statuses.id
+              WHERE statuses.user_id IN ?
+              AND statuses.workspace_id = ?
+              AND DATE(statuses.posted_at) <= DATE(?)
+              AND DATE(statuses.posted_at) >= DATE(?)
+              GROUP BY urls.url, urls.final_url, urls.title, urls.http_status, urls.content_type
+              ORDER BY count DESC", user_ids, current_workspace.id, since, before].all
+          else
+            urls = database["SELECT urls.url, urls.final_url, urls.title, urls.http_status, urls.content_type, count(statuses_urls.*) AS count
+              FROM urls
+              INNER JOIN statuses_urls
+                ON statuses_urls.url_id = urls.id
+              INNER JOIN statuses
+                ON statuses_urls.status_id = statuses.id
+              WHERE statuses.workspace_id = ?
+              AND DATE(statuses.posted_at) <= DATE(?)
+              AND DATE(statuses.posted_at) >= DATE(?)
+              GROUP BY urls.url, urls.final_url, urls.title, urls.http_status, urls.content_type
+              ORDER BY count DESC", current_workspace.id, before, since].all
+          end
+          if urls.count.zero?
+            error("There are no URLs to display")
+            return false
+          end
+          text = ""
+          urls.each do |url|
+            next if option_setting("MIN_SHARE_COUNT") && url[:count] <= option_setting("MIN_SHARE_COUNT")
+            text += make_url_summary_output(url) + "\n#{Birdwatcher::Console::LINE_SEPARATOR}\n\n"
+          end
+          page_text(text)
+        end
+      end
+    end
+  end
+end

data/lib/birdwatcher/modules/users/activity_plot.rb

@@ -0,0 +1,62 @@
+module Birdwatcher
+  module Modules
+    module Statuses
+      class ActivityPlot < Birdwatcher::Module
+        self.meta = {
+          :name        => "Activity Plot",
+          :description => "Generates punchcard plot of a user's activity",
+          :author      => "Michael Henriksen <michenriksen@neomailbox.ch>",
+          :options     => {
+            "DEST" => {
+              :value       => nil,
+              :description => "Destination file",
+              :required    => true
+            },
+            "USER" => {
+              :value       => nil,
+              :description => "Screen name of user to analyze",
+              :required    => true
+            },
+            "ONLY_REPLIES" => {
+              :value       => false,
+              :description => "Only plot when the user replies to other users",
+              :required    => false,
+              :boolean     => true
+            }
+          }
+        }
+
+        def self.info
+<<-INFO
+The Activity Plot module can generate a punchcard plot of when a user is the
+most engaged with Twitter. The plot can be used to find the most likely time
+(day and hour) where a user will engage with Twitter content.
+
+The generated file is in PNG format.
+INFO
+        end
+
+        def run
+          if !user = current_workspace.users_dataset.first(:screen_name => option_setting("USER"))
+            error("User #{screen_name.bold} was not found in workspace")
+            return false
+          end
+          if option_setting("ONLY_REPLIES")
+            timestamps = user.statuses_dataset.where("text LIKE '@%'").map(&:posted_at)
+          else
+            timestamps = user.statuses.map(&:posted_at)
+          end
+          if timestamps.empty?
+            error("There are no statuses to process")
+            return false
+          end
+          punchcard = Birdwatcher::Punchcard.new(timestamps)
+          task("Generating activity plot from #{timestamps.count.to_s.bold} statuses...") do
+            punchcard.generate(option_setting("DEST"))
+          end
+          info("Activity plot written to #{option_setting('DEST').bold}")
+        end
+      end
+    end
+  end
+end

data/lib/birdwatcher/modules/users/import.rb

@@ -0,0 +1,61 @@
+module Birdwatcher
+  module Modules
+    module Users
+      class Import < Birdwatcher::Module
+        self.meta = {
+          :name        => "User Importer",
+          :description => "Import users from a file containing screen names.",
+          :author      => "Michael Henriksen <michenriksen@neomailbox.ch>",
+          :options     => {
+            "FILE" => {
+              :value       => nil,
+              :description => "File to read screen names from.",
+              :required    => true
+            }
+          }
+        }
+
+        def self.info
+<<-INFO
+The User Importer module is a simple module to add a large number of users to
+the currently active workspace by parsing a file containing screen names.
+
+The file is expected to contain one screen name per line, without the @ sign or
+https://twitter.com/ in front of them.
+INFO
+        end
+
+        def run
+          filename = File.expand_path(option_setting("FILE"))
+          if !File.exists?(filename)
+            error("File #{filename.bold} does not exist")
+            return false
+          end
+          if !File.readable?(filename)
+            error("File #{filename.bold} is not readable")
+            return false
+          end
+          threads = thread_pool
+          File.read(filename).each_line do |screen_name|
+            threads.process do
+              begin
+                screen_name.strip!
+                next if screen_name.empty?
+                if current_workspace.users_dataset.first(:screen_name => screen_name)
+                  info("User #{screen_name.bold} is already in the workspace")
+                  next
+                end
+                user = twitter_client.user(screen_name)
+                save_user(user)
+                info("Added #{screen_name.bold} to workspace")
+              rescue Twitter::Error::NotFound
+                error("There is no user with screen name: #{screen_name.bold}")
+              end
+            end
+          end
+          threads.shutdown
+        end
+      end
+    end
+  end
+end

data/lib/birdwatcher/modules/users/influence_graph.rb

@@ -0,0 +1,93 @@
+module Birdwatcher
+  module Modules
+    module Users
+      class InfluenceGraph < Birdwatcher::Module
+        self.meta = {
+          :name        => "Influence Graph",
+          :description => "Graphs the influence between users from Klout",
+          :author      => "Michael Henriksen <michenriksen@neomailbox.ch>",
+          :options     => {
+            "DEST" => {
+              :value       => nil,
+              :description => "Destination file",
+              :required    => true
+            },
+            "USERS" => {
+              :value       => nil,
+              :description => "Space-separated list of screen names (all users if empty)",
+              :required    => false
+            },
+            "FORMAT" => {
+              :value       => "png",
+              :description => "Destination file format (any format supported by Graphviz)",
+              :required    => true
+            }
+          }
+        }
+
+        def self.info
+<<-INFO
+The Influence Graph module generates an influence graph between users in the
+currently active workspace. The graph can be used to identify who each user is
+being influenced by as well as who each user influences.
+
+The influence information is retrieved by the #{'users/klout_influence'.bold} so
+be sure to run that module before running this one.
+
+The generated graph is in PNG format.
+INFO
+        end
+
+        def run
+          if !GraphViz::Constants::FORMATS.include?(option_setting("FORMAT"))
+            error("Unsupported format: #{option_setting('FORMAT').bold}")
+            return false
+          end
+          if screen_names = option_setting("USERS")
+            users = current_workspace.users_dataset
+              .where("screen_name IN ?", screen_names.split(" ").map(&:strip))
+              .order(:screen_name)
+              .eager(:influencers, :influencees)
+          else
+            users = current_workspace.users_dataset.order(:screen_name).eager(:influencers, :influencees)
+          end
+          if users.empty?
+            error("There are no users to process")
+            return false
+          end
+          graph = GraphViz.new(:G, :type => :digraph)
+          users_in_workspace = current_workspace.users.map(&:screen_name)
+          nodes = {}
+          influences = {}
+          users.each do |user|
+            influences[user.screen_name] ||= []
+            influences[user.screen_name] += user.influencees.select { |i| users_in_workspace.include?(i.screen_name) }.map(&:screen_name)
+            user.influencers.select { |i| users_in_workspace.include?(i.screen_name) }.map(&:screen_name).each do |influencer|
+              influences[influencer] ||= []
+              influences[influencer] << user.screen_name unless influences[influencer].include?(user.screen_name)
+            end
+          end
+          influences.each_pair do |user, influence|
+            influence.uniq!
+            next if influence.empty?
+            nodes[user] ||= graph.add_nodes(user)
+            influence.each do |influencee|
+              if influences[influencee] && influences[influencee].include?(user)
+                direction = "both"
+                influences[influencee].delete(user)
+              else
+                direction = "forward"
+              end
+              nodes[influencee] ||= graph.add_nodes(influencee)
+              graph.add_edges(nodes[user], nodes[influencee], :color => "lightblue", :fontcolor => "cornflowerblue", :dir => direction, :arrowhead => "normal")
+            end
+          end
+          task("Outputting graph...") do
+            graph.output(option_setting("FORMAT") => option_setting("DEST"))
+          end
+          info("Graph written to #{option_setting('DEST').bold}")
+        end
+      end
+    end
+  end
+end