birdwatcher 0.1.0

data/lib/birdwatcher/klout_client.rb

@@ -0,0 +1,83 @@
+module Birdwatcher
+  class KloutClient < Birdwatcher::HttpClient
+    base_uri "https://api.klout.com/v2"
+
+    # Class initializer
+    #
+    # @param api_key [String] Klout API key
+    # @param options Http client options
+    # @see Birdwatcher::HttpClient
+    def initialize(api_key, options = {})
+      @api_key = api_key
+      @options = {
+        :headers => {
+          "User-Agent" => "Birdwatcher v#{Birdwatcher::VERSION}",
+          "Accept"     => "application/json"
+        }
+      }.merge(options)
+    end
+
+    # Get Klout ID of a Twitter user
+    #
+    # @param screen_name [String] Twitter screen name
+    # @return [String] Klout ID or nil
+    # @see https://klout.com/s/developers/v2#identities
+    def get_id(screen_name)
+      response = do_get("/identity.json/twitter?screenName=#{url_encode(screen_name)}&key=#{url_encode(@api_key)}")
+      if response.status == 200
+        JSON.parse(response.body)["id"]
+      end
+    end
+
+    # Get Klout score of a user
+    #
+    # @param klout_id [String]
+    # @return [Numeric] Klout score or nil
+    # @see https://klout.com/s/developers/v2#scores
+    def get_score(klout_id)
+      response = do_get("/user.json/#{klout_id}/score?key=#{url_encode(@api_key)}")
+      if response.status == 200
+        JSON.parse(response.body)["score"]
+      end
+    end
+
+    # Get Klout topics of a user
+    #
+    # @param klout_id [String]
+    # @return [Array] Topics
+    # @see https://klout.com/s/developers/v2#topic
+    def get_topics(klout_id)
+      response = do_get("/user.json/#{klout_id}/topics?key=#{url_encode(@api_key)}")
+      if response.status == 200
+        JSON.parse(response.body).map { |t| t["displayName"] }
+      end
+    end
+
+    # Get Klout influence graph of a user
+    #
+    # @param klout_id [String]
+    # @return [Hash] +:influencers:+ contains screen names of influencers, +:influencees+ contains screen names of influencees
+    # @see https://klout.com/s/developers/v2#influence
+    def get_influence(klout_id)
+      response = do_get("/user.json/#{klout_id}/influence?key=#{url_encode(@api_key)}")
+      if response.status == 200
+        body = JSON.parse(response.body)
+        {
+          :influencers => body["myInfluencers"].map { |i| i["entity"]["payload"]["nick"] },
+          :influencees => body["myInfluencees"].map { |i| i["entity"]["payload"]["nick"] }
+        }
+      end
+    end
+
+    private
+
+    # URL encode a string
+    # @private
+    #
+    # @param string [String]
+    # @return [String] URL encoded string
+    def url_encode(string)
+      CGI.escape(string.to_s)
+    end
+  end
+end

data/lib/birdwatcher/kml.rb

@@ -0,0 +1,125 @@
+module Birdwatcher
+  # KML Document generator
+  #
+  # KML is a file format used to display geographic data in an Earth browser
+  # such as Google Earth. You can create KML files to pinpoint locations, add
+  # image overlays, and expose rich data in new ways. KML is an international
+  # standard maintained by the Open Geospatial Consortium, Inc. (OGC).
+  #
+  # This class supports generating basic KML documents with Placemarks and Folders.
+  #
+  # @note Attribute values ARE NOT automatically escaped. All values will have to be given in an HTML escaped fashion if there is a risk that they might contain unexpected or dangerous HTML.
+  # @see https://developers.google.com/kml/
+  class KML
+    # KML document header
+    DOCUMENT_HEADER =
+<<-HEAD
+<?xml version="1.0" encoding="UTF-8"?>
+<kml xmlns="http://www.opengis.net/kml/2.2">
+<Document>
+HEAD
+
+    # KML document footer
+    DOCUMENT_FOOTER =
+<<-FOOT
+</Document>
+</kml>
+FOOT
+
+    class Error < StandardError; end
+    class UnknownFolderError < Birdwatcher::KML::Error; end
+
+    # Class initializer
+    #
+    # @param attributes [Hash] Document attributes
+    # @see https://developers.google.com/kml/documentation/kmlreference#document
+    def initialize(attributes = {})
+      @attributes = attributes
+      @folders    = {}
+      @placemarks = []
+    end
+
+    # Add a Placemark
+    #
+    # @param attributes [Hash] Placemark attributes
+    # @see https://developers.google.com/kml/documentation/kmlreference#placemark
+    def add_placemark(attributes)
+      @placemarks << attributes
+    end
+
+    # Add a Folder
+    #
+    # @param id [String] Folder ID
+    # @param attributes [Hash] Folder attributes
+    #
+    # @see https://developers.google.com/kml/documentation/kmlreference#folder
+    def add_folder(id, attributes)
+      @folders[id] = {
+        :placemarks => []
+      }.merge(attributes)
+    end
+
+    # Add a Placemark to a Folder
+    #
+    # @param folder_id [String]
+    # @param attributes [Hash] Placemark attributes
+    #
+    # @raise [Birdwatcher::KML::UnknownFolderError] if folder doesn't exist
+    # @see https://developers.google.com/kml/documentation/kmlreference#placemark
+    def add_placemark_to_folder(folder_id, attributes)
+      fail(UnknownFolderError, "There is no folder with id: #{folder_id}") unless @folders.key?(folder_id)
+      @folders[folder_id][:placemarks] << attributes
+    end
+
+    # Generate the KML document
+    #
+    # @return the final KML document
+    def generate
+      output = generate_document_header
+      @folders.each_pair { |id, attributes| output += generate_folder(id, attributes) }
+      output += @placemarks.map { |p| generate_placemark(p) }.join
+      output += generate_document_footer
+    end
+
+    private
+
+    # Generate document header
+    # @private
+    def generate_document_header
+      header = DOCUMENT_HEADER
+      @attributes.each_pair { |k, v| header += "<#{k}>#{escape(v)}</#{k}>\n" }
+      header
+    end
+
+    # Generate document footer
+    # @private
+    def generate_document_footer
+      DOCUMENT_FOOTER
+    end
+
+    # Generate Placemark element
+    # @private
+    def generate_placemark(attributes)
+      placemark = attributes.key?(:id) ? "<Placemark id='#{escape(attributes[:id])}'>" : "<Placemark>"
+      attributes.delete(:id)
+      attributes.each_pair { |k, v| placemark += "<#{k}>#{v}</#{k}>\n" }
+      placemark += "</Placemark>\n"
+    end
+
+    # Generate Folder element
+    # @private
+    def generate_folder(id, attributes)
+      placemarks = attributes.delete(:placemarks)
+      folder = "<Folder id='#{escape(id)}'>"
+      attributes.each_pair { |k, v| folder += "<#{k}>#{escape(v)}</#{k}>\n" }
+      folder += placemarks.map { |p| generate_placemark(p) }.join
+      folder += "</Folder>\n"
+    end
+
+    # HTML escape a string
+    # @private
+    def escape(string)
+      CGI.escapeHTML(string.to_s)
+    end
+  end
+end

data/lib/birdwatcher/module.rb

@@ -0,0 +1,253 @@
+module Birdwatcher
+  class Module
+    class Error < StandardError; end
+    class InvalidMetadataError < Error; end
+    class MetadataNotSetError < Error; end
+    class UnknownOptionError < Error; end
+
+    include Birdwatcher::Concerns::Core
+    include Birdwatcher::Concerns::Util
+    include Birdwatcher::Concerns::Outputting
+    include Birdwatcher::Concerns::Presentation
+    include Birdwatcher::Concerns::Persistence
+    include Birdwatcher::Concerns::Concurrency
+
+    # Path to modules directory
+    # @private
+    MODULE_PATH = File.join(File.dirname(__FILE__), "modules").freeze
+
+    # Get the module's file path
+    # @private
+    def self._file_path
+      @_file_path
+    end
+
+    # Set the module's file path
+    # @private
+    #
+    # @param path [String] file path
+    def self._file_path=(path)
+      @_file_path = path
+    end
+
+    # Automatically set the module file path
+    # @private
+    def self.inherited(k)
+      k._file_path = caller.first[/^[^:]+/]
+    end
+
+    # Get the module's meta data
+    # @private
+    #
+    # @return [Hash] meta data
+    # @raise [Birdwatcher::Model::MetadataNotSetError] if meta data has not been set
+    def self.meta
+      @meta || fail(MetadataNotSetError, "Metadata has not been set")
+    end
+
+    # Set the module's meta data
+    #
+    # @param meta [Hash] meta data
+    #
+    # The module's meta data is used by Birdwatcher to provide the user with
+    # useful information such as name, a short description of what it does as
+    # well as the author of the module in case they have any questions, etc.
+    #
+    # The meta data MUST be a hash and MUST contain at least the following keys:
+    # * +:name+: The module's name (e.g. User Importer)
+    # * +:description+: A short description of what the module can do
+    # * +:author+: Your name and email (e.g. John Doe <john@doe.com>)
+    # * +:options+: A hash of options for the module
+    #
+    # The +:options+ meta data key MUST be a Hash where each key is the option name
+    # in UPPERCASE. The value MUST be a Hash and MUST contain at least the following
+    # keys:
+    # * +:value+: The default value of the option setting (set to +nil+ if none)
+    # * +:description+: A short description of the option setting
+    # * +:required+: Set to +true+ if the option setting is required to be set
+    #
+    # If the option setting is a boolean flag, the +:boolean+ key can be set to
+    # +true+ to have Birdwatcher automatically parse "truthy" and "falsy" values
+    # (e.g. "true", "1", "yes", "no", "0", etc) into boolean true or false
+    #
+    # If an option setting's +:required+ key is set to +true+, Birdwatcher will
+    # automatically prevent running of the module if any of those option settings
+    # contain +nil+ (have not been set).
+    #
+    # @example Example meta data:
+    #     self.meta = {
+    #       :name        => "User Importer",
+    #       :description => "Import users from a file containing screen names",
+    #       :author      => "Michael Henriksen <michenriksen@neomailbox.ch>",
+    #       :options     => {
+    #         "FILE" => {
+    #           :value       => nil,
+    #           :description => "File to read screen names from.",
+    #           :required    => true
+    #         }
+    #       }
+    #     }
+    def self.meta=(meta)
+      validate_metadata(meta)
+      @meta = meta
+    end
+
+    # Get a module by it's path
+    # @private
+    #
+    # @param path [String] Module's short path
+    #
+    # @return [Birdwatcher::Module] descendant
+    def self.module_by_path(path)
+      modules[path]
+    end
+
+    # Get module short paths
+    # @private
+    def self.module_paths
+      modules.keys
+    end
+
+    # Get the module's short path
+    # @private
+    def self.path
+      @_file_path.gsub("#{MODULE_PATH}/", "").gsub(".rb", "")
+    end
+
+    # The module's detailed information and usage
+    #
+    # @abstract
+    #
+    # This method can be overwritten by modules to provide additional information
+    # and usage to the user. The method will be called when the user envokes the
+    # +show info+ on the module.
+    #
+    # The method must return a string.
+    #
+    # @return [String] additional module information
+    def self.info; end
+
+    # Get all Birdwatcher::Module descendants
+    # @private
+    #
+    # @return [Array] module classes
+    def self.descendants
+      ObjectSpace.each_object(Class).select { |klass| klass < self }
+    end
+
+    # Get all Birdwatcher modules sorted by their short path
+    # @private
+    #
+    # @return [Hash] module classes where the key is the module's short path
+    def self.modules
+      if !@modules
+        @modules = {}
+        descendants.each do |descendant|
+          @modules[descendant.path] = descendant
+        end
+      end
+      @modules
+    end
+
+    # Execute a module and catch any exceptions raised
+    # @private
+    #
+    # Calls the module's {run} method if options are valid and catches any
+    # exceptions raised to display an error to the user.
+    def execute
+      validate_options && run
+    rescue => e
+      error("#{e.class}".bold + ": #{e.message}")
+      puts e.backtrace.join("\n")
+    end
+
+    # The module's run method
+    #
+    # @abstract
+    #
+    # The run method must be overwritten by modules to perform the actual work.
+    # The method is called when the user envokes the +run+ command in the
+    # Birdwatcher console.
+    #
+    # If the module fails to run for whatever reason, e.g. insufficient data, the
+    # method should return +false+.
+    def run
+      fail NotImplementedError, "Modules must implement #run method"
+    end
+
+    protected
+
+    # Validate option settings
+    # @private
+    #
+    # @return [Boolean] true if meta data is valid and false otherwise
+    def validate_options
+      options.each_pair do |key, value|
+        if value[:required] && value[:value].nil?
+          error("Setting for required option has not been set: #{key.bold}")
+          return false
+        end
+      end
+    end
+
+    # Validate module meta data
+    # @private
+    #
+    # @param meta [Hash] meta data
+    #
+    # @raise [Birdwatcher::Module::InvalidMetadataError] if meta data is not valid.
+    def self.validate_metadata(meta)
+      fail InvalidMetadataError, "Metadata is not a hash" unless meta.is_a?(Hash)
+      fail InvalidMetadataError, "Metadata is empty" if meta.empty?
+      fail InvalidMetadataError, "Metadata is missing key: name" unless meta.key?(:name)
+      fail InvalidMetadataError, "Metadata is missing key: description" unless meta.key?(:description)
+      fail InvalidMetadataError, "Metadata is missing key: author" unless meta.key?(:author)
+      fail InvalidMetadataError, "Metadata is missing key: options" unless meta.key?(:options)
+      fail InvalidMetadataError, "Metadata name is not a string" unless meta[:name].is_a?(String)
+      fail InvalidMetadataError, "Metadata description is not a string" unless meta[:description].is_a?(String)
+      fail InvalidMetadataError, "Metadata author is not a string" unless meta[:author].is_a?(String)
+      validate_metadata_options(meta[:options])
+    end
+
+    # Validate meta data module options
+    # @private
+    #
+    # @param options [Hash] options
+    #
+    # Automatically called by {validate_metadata}
+    #
+    # @raise [Birdwatcher::Module::InvalidMetadataError] if options hash is not valid.
+    def self.validate_metadata_options(options)
+      fail InvalidMetadataError, "Metadata options is not a hash" unless options.is_a?(Hash)
+      options.each_pair do |key, value|
+        fail("Option key #{key} must be all uppercase") unless (key == key.upcase)
+        fail("Option value for #{key} is not a hash") unless value.is_a?(Hash)
+        fail("Option value for #{key} is missing key: value") unless value.key?(:value)
+        fail("Option value for #{key} is missing key: description") unless value.key?(:description)
+        fail("Option value for #{key} is missing key: required") unless value.key?(:required)
+      end
+    end
+
+    # Get the module's options hash
+    # @private
+    #
+    # @return [Hash] options meta data hash.
+    def options
+      self.class.meta[:options]
+    end
+
+    # Get an option setting
+    #
+    # @example getting option settings
+    #     option_setting("DEST")
+    #     option_setting("USERS")
+    #
+    # @return option setting
+    # @raise [Birdwatcher::Module::UnknownOptionError] if option is unknown
+    def option_setting(option)
+      option = option.to_s.upcase
+      fail UnknownOptionError, "Unknown module option: #{option}" unless options.keys.include?(option)
+      options[option][:value]
+    end
+  end
+end

data/lib/birdwatcher/modules/statuses/kml.rb

@@ -0,0 +1,106 @@
+module Birdwatcher
+  module Modules
+    module Statuses
+      class Kml < Birdwatcher::Module
+        self.meta = {
+          :name        => "KML Document",
+          :description => "Creates a KML document of statuses with Geo locations",
+          :author      => "Michael Henriksen <michenriksen@neomailbox.ch>",
+          :options     => {
+            "DEST" => {
+              :value       => nil,
+              :description => "Destination file",
+              :required    => true
+            },
+            "USERS" => {
+              :value       => nil,
+              :description => "Space-separated list of screen names (all users if empty)",
+              :required    => false
+            }
+          }
+        }
+
+        def self.info
+<<-INFO
+KML is a file format used to display geographic data in an Earth browser such as
+Google Earth. You can create KML files to pinpoint locations, add image overlays,
+and expose rich data in new ways. KML is an international standard maintained by
+the Open Geospatial Consortium, Inc. (OGC).
+
+This module can generate a KML document containing all statuses with geo information
+which can be loaded into an application like Google Earth to browse and analyze
+statuses.
+
+The module supports mapping statuses from all or specific users.
+INFO
+        end
+
+        def run
+          if option_setting("USERS")
+            users = current_workspace.users_dataset
+              .where("screen_name IN ?", option_setting("USERS").split(" ").map(&:strip))
+              .order(:screen_name)
+          else
+            users = current_workspace.users_dataset.order(:screen_name)
+          end
+          if users.empty?
+            error("There are no users to process")
+            return false
+          end
+          kml_document = Birdwatcher::KML.new(
+            :name => "Statuses with geo locations"
+          )
+          users.each do |user|
+            statuses = user.statuses_dataset.where(:geo => true).order(Sequel.desc(:posted_at)).eager(:user)
+            if statuses.count.zero?
+              warn("User #{user.screen_name.bold} has no statuses with geo location; skipping")
+              next
+            end
+            kml_document.add_folder(user.screen_name,
+              :name        => "#{user.name} (@#{user.screen_name})",
+              :description => "Statuses from #{user.screen_name}"
+            )
+            statuses.each do |status|
+              kml_document.add_placemark_to_folder(user.screen_name,
+                :id          => status.twitter_id,
+                :name        => "@#{escape_html(user.screen_name)}, #{format_date(status.posted_at)}",
+                :description => make_status_description(status),
+                :Snippet     => escape_html(excerpt(status.text, 80)),
+                :Style       => "<Icon><href>#{escape_html(user.profile_image_url)}</href></Icon>",
+                :Point       => "<coordinates>#{escape_html(status.latitude)},#{escape_html(status.longitude)}</coordinates>",
+                :address     => "#{escape_html(status.place_name)}, #{escape_html(status.place_country)}",
+                :TimeStamp   => escape_html(status.posted_at.strftime('%Y-%m-%dT%l:%M:%S%z'))
+              )
+            end
+            info "Added #{pluralize(statuses.count, 'status', 'statuses')} from #{user.screen_name.bold}"
+          end
+          File.write(option_setting("DEST"), kml_document.generate)
+          info("Wrote KML document to #{option_setting('DEST').bold}")
+        end
+
+        private
+
+        def make_status_description(status)
+          "<![CDATA[\n" +
+          "  <table>\n" +
+          "    <tr>\n" +
+          "      <td style='vertical-align:top'>\n" +
+          "        <img src='#{escape_html(status.user.profile_image_url)}' alt='#{escape_html(status.user.screen_name)}' />\n" +
+          "      </td>\n" +
+          "      <td>\n" +
+          "        <strong>#{escape_html(status.user.name)}</strong> <span style='color: #8899a6'>@#{escape_html(status.user.screen_name)} &middot; <a href='https://twitter.com/#{escape_html(status.user.screen_name)}/status/#{escape_html(status.twitter_id)}' style='color: inherit'>#{format_date(status.posted_at)}</a></span><br />\n" +
+          "          <p style='font-size: 16px'>#{escape_html(status.text)}</p>\n" +
+          "          <p style='font-style: italic'>#{escape_html(status.place_name)}, #{escape_html(status.place_country)}</p>\n" +
+          "      </td>\n" +
+          "    </tr>\n" +
+          "  </table>\n" +
+          "]]>\n"
+        end
+
+        def format_date(time)
+          time.strftime("%b %d, %Y")
+        end
+      end
+    end
+  end
+end

data/lib/birdwatcher/modules/statuses/sentiment.rb

@@ -0,0 +1,77 @@
+module Birdwatcher
+  module Modules
+    module Statuses
+      class Sentiment < Birdwatcher::Module
+        self.meta = {
+          :name        => "Status Sentiment Analysis",
+          :description => "Enrich statuses with sentiment score",
+          :author      => "Michael Henriksen <michenriksen@neomailbox.ch>",
+          :options     => {
+            "THREADS" => {
+              :value       => Birdwatcher::Concerns::Concurrency::DEFAULT_THREAD_POOL_SIZE,
+              :description => "Number of concurrent threads",
+              :required    => false
+            }
+          }
+        }
+
+        def self.info
+<<-INFO
+The Status Sentiment Analysis module can calculate the rough sentiment of statuses
+in the workspace. Each status will get a sentiment score of Negative, Positive
+or Neutral.
+
+Please note that sentiment analysis is not very accurate and should always be
+manually reviewed for serious work.
+INFO
+        end
+
+        def run
+          statuses = current_workspace.statuses_dataset.where(:sentiment => nil)
+          if statuses.empty?
+            error("There are no statuses to analyze")
+            return false
+          end
+          analyser = Sentimental.new
+          threads  = thread_pool(option_setting("THREADS").to_i)
+          task("Training the sentiment analyzer...") do
+            analyser.load_defaults
+          end
+          statuses.each do |status|
+            threads.process do
+              begin
+                text = sanitize_text(status.text)
+                sentiment = analyser.sentiment(text)
+                case sentiment
+                when :positive
+                  info("Positive: ".bold.light_green + Birdwatcher::Util.excerpt(status.text, 80))
+                when :negative
+                  info("Negative: ".bold.light_red + Birdwatcher::Util.excerpt(status.text, 80))
+                else
+                  info(" Neutral: ".bold + Birdwatcher::Util.excerpt(status.text, 80))
+                end
+                status.sentiment = sentiment.to_s
+                status.save
+              rescue => e
+                error("Sentiment analysis for status #{status.id.bold} failed (#{e.class})")
+              end
+            end
+          end
+          threads.shutdown
+        end
+
+        private
+
+        def sanitize_text(text)
+          text.split(" ").map(&:strip).delete_if do |word|
+            word.start_with?("@") ||
+            word.start_with?(".@")
+            word.start_with?("#") ||
+            word.downcase.start_with?("http") ||
+            %w(rt oh).include?(word.downcase)
+          end.join(" ")
+        end
+      end
+    end
+  end
+end