birdwatcher 0.1.0

data/db/migrations/001_create_workspaces.rb

@@ -0,0 +1,11 @@
+Sequel.migration do
+  change do
+    create_table(:workspaces) do
+      primary_key :id
+      String :name, :index => true, :unique => true
+      String :description
+      DateTime :updated_at
+      DateTime :created_at
+    end
+  end
+end

data/db/migrations/002_create_users.rb

@@ -0,0 +1,29 @@
+Sequel.migration do
+  change do
+    create_table(:users) do
+      primary_key :id
+      foreign_key :workspace_id, :workspaces, :on_delete => :cascade, :index => true
+      String :twitter_id, :index => true
+      String :screen_name, :index => true
+      String :name
+      String :location
+      String :description
+      String :url
+      String :profile_image_url
+      Integer :followers_count, :index => true
+      Integer :friends_count, :index => true
+      Integer :listed_count, :index => true
+      Integer :favorites_count, :index => true
+      Integer :statuses_count, :index => true
+      Integer :utc_offset
+      String :timezone
+      Boolean :geo_enabled, :index => true
+      Boolean :verified, :index => true
+      String :lang, :index => true
+      String :klout_id
+      Integer :klout_score, :index => true
+      DateTime :updated_at, :index => true
+      DateTime :created_at, :index => true
+    end
+  end
+end

data/db/migrations/003_create_statuses.rb

@@ -0,0 +1,28 @@
+Sequel.migration do
+  change do
+    create_table(:statuses) do
+      primary_key :id
+      foreign_key :workspace_id, :workspaces, :on_delete => :cascade, :index => true
+      foreign_key :user_id, :users, :on_delete => :cascade, :index => true
+      String :twitter_id, :index => true
+      String :text, :index => true
+      String :source
+      Boolean :retweet, :index => true
+      Boolean :geo, :index => true
+      String :longitude
+      String :latitude
+      String :place_type
+      String :place_name
+      String :place_country_code
+      String :place_country
+      Integer :favorite_count, :index => true
+      Integer :retweet_count, :index => true
+      String :sentiment, :index => true
+      Boolean :possibly_sensitive, :index => true
+      String :lang, :index => true
+      DateTime :posted_at, :index => true
+      DateTime :updated_at, :index => true
+      DateTime :created_at, :index => true
+    end
+  end
+end

data/db/migrations/004_create_mentions.rb

@@ -0,0 +1,13 @@
+Sequel.migration do
+  change do
+    create_table(:mentions) do
+      primary_key :id
+      foreign_key :workspace_id, :workspaces, :on_delete => :cascade, :index => true
+      String :twitter_id, :index => true
+      String :screen_name, :index => true
+      String :name
+      DateTime :updated_at
+      DateTime :created_at, :index => true
+    end
+  end
+end

data/db/migrations/005_create_mentions_statuses.rb

@@ -0,0 +1,8 @@
+Sequel.migration do
+  change do
+    create_table(:mentions_statuses) do
+      foreign_key :mention_id, :mentions, :on_delete => :cascade, :index => true
+      foreign_key :status_id, :statuses, :on_delete => :cascade, :index => true
+    end
+  end
+end

data/db/migrations/006_create_hashtags.rb

@@ -0,0 +1,11 @@
+Sequel.migration do
+  change do
+    create_table(:hashtags) do
+      primary_key :id
+      foreign_key :workspace_id, :workspaces, :on_delete => :cascade, :index => true
+      String :tag, :index => true
+      DateTime :updated_at
+      DateTime :created_at, :index => true
+    end
+  end
+end

data/db/migrations/007_create_hashtags_statuses.rb

@@ -0,0 +1,8 @@
+Sequel.migration do
+  change do
+    create_table(:hashtags_statuses) do
+      foreign_key :hashtag_id, :hashtags, :on_delete => :cascade, :index => true
+      foreign_key :status_id, :statuses, :on_delete => :cascade, :index => true
+    end
+  end
+end

data/db/migrations/008_create_urls.rb

@@ -0,0 +1,16 @@
+Sequel.migration do
+  change do
+    create_table(:urls) do
+      primary_key :id
+      foreign_key :workspace_id, :workspaces, :on_delete => :cascade, :index => true
+      String :url, :index => true
+      String :final_url, :index => true
+      Integer :http_status, :index => true
+      String :content_type, :index => true
+      String :title, :index => true
+      DateTime :crawled_at, :index => true
+      DateTime :updated_at, :index => true
+      DateTime :created_at, :index => true
+    end
+  end
+end

data/db/migrations/009_create_statuses_urls.rb

@@ -0,0 +1,8 @@
+Sequel.migration do
+  change do
+    create_table(:statuses_urls) do
+      foreign_key :url_id, :urls, :on_delete => :cascade, :index => true
+      foreign_key :status_id, :statuses, :on_delete => :cascade, :index => true
+    end
+  end
+end

data/db/migrations/010_create_klout_topics.rb

@@ -0,0 +1,10 @@
+Sequel.migration do
+  change do
+    create_table(:klout_topics) do
+      primary_key :id
+      foreign_key :workspace_id, :workspaces, :on_delete => :cascade, :index => true
+      String :topic, :index => true
+      DateTime :created_at, :index => true
+    end
+  end
+end

data/db/migrations/011_create_klout_topics_users.rb

@@ -0,0 +1,8 @@
+Sequel.migration do
+  change do
+    create_table(:klout_topics_users) do
+      foreign_key :user_id, :users, :on_delete => :cascade, :index => true
+      foreign_key :klout_topic_id, :klout_topics, :on_delete => :cascade, :index => true
+    end
+  end
+end

data/db/migrations/012_create_influencers.rb

@@ -0,0 +1,10 @@
+Sequel.migration do
+  change do
+    create_table(:influencers) do
+      primary_key :id
+      foreign_key :workspace_id, :workspaces, :on_delete => :cascade, :index => true
+      String :screen_name, :index => true
+      DateTime :created_at, :index => true
+    end
+  end
+end

data/db/migrations/013_create_influencers_users.rb

@@ -0,0 +1,8 @@
+Sequel.migration do
+  change do
+    create_table(:influencers_users) do
+      foreign_key :user_id, :users, :on_delete => :cascade, :index => true
+      foreign_key :influencer_id, :influencers, :on_delete => :cascade, :index => true
+    end
+  end
+end

data/db/migrations/014_create_influencees.rb

@@ -0,0 +1,10 @@
+Sequel.migration do
+  change do
+    create_table(:influencees) do
+      primary_key :id
+      foreign_key :workspace_id, :workspaces, :on_delete => :cascade, :index => true
+      String :screen_name, :index => true
+      DateTime :created_at, :index => true
+    end
+  end
+end

data/db/migrations/015_create_influencees_users.rb

@@ -0,0 +1,8 @@
+Sequel.migration do
+  change do
+    create_table(:influencees_users) do
+      foreign_key :user_id, :users, :on_delete => :cascade, :index => true
+      foreign_key :influencee_id, :influencees, :on_delete => :cascade, :index => true
+    end
+  end
+end

data/exe/birdwatcher

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+require "birdwatcher"
+require "readline"
+
+trap "SIGINT" do
+  print "\n"
+  Birdwatcher::Console.instance.warn("Caught interrupt; Exiting.")
+  exit
+end
+
+Birdwatcher::Console.instance.start!

data/lib/birdwatcher/command.rb

@@ -0,0 +1,78 @@
+module Birdwatcher
+  class Command
+    class Error < StandardError; end
+    class InvalidMetadataError < Error; end
+    class MetadataNotSetError < Error; end
+
+    ARGUMENT_SEPARATOR = " ".freeze
+
+    attr_reader :arguments
+
+    include Birdwatcher::Concerns::Core
+    include Birdwatcher::Concerns::Util
+    include Birdwatcher::Concerns::Outputting
+    include Birdwatcher::Concerns::Presentation
+    include Birdwatcher::Concerns::Persistence
+    include Birdwatcher::Concerns::Concurrency
+
+    def self.meta
+      @meta || fail(MetadataNotSetError, "Metadata has not been set")
+    end
+
+    def self.meta=(meta)
+      validate_metadata(meta)
+      @meta = meta
+    end
+
+    def self.detailed_usage; end
+
+    def self.descendants
+      ObjectSpace.each_object(Class).select { |klass| klass < self }
+    end
+
+    def self.has_name?(name)
+      meta[:names].include?(name)
+    end
+
+    def self.auto_completion_strings
+      (meta[:names] + auto_completion).uniq
+    end
+
+    def self.auto_completion
+      []
+    end
+
+    def execute(argument_line)
+      @arguments = argument_line.to_s.split(ARGUMENT_SEPARATOR).map { |a| a.to_s.strip }
+      run
+    rescue => e
+      error("#{e.class}".bold + ": #{e.message}")
+      puts e.backtrace.join("\n")
+    end
+
+    protected
+
+    def run
+      fail NotImplementedError, "Commands must implement #run method"
+    end
+
+    def arguments?
+      !arguments.empty?
+    end
+
+    def commands
+      Birdwatcher::Command.descendants
+    end
+
+    def self.validate_metadata(meta)
+      fail InvalidMetadataError, "Metadata is not a hash" unless meta.is_a?(Hash)
+      fail InvalidMetadataError, "Metadata is empty" if meta.empty?
+      fail InvalidMetadataError, "Metadata is missing key: description" unless meta.key?(:description)
+      fail InvalidMetadataError, "Metadata is missing key: names" unless meta.key?(:names)
+      fail InvalidMetadataError, "Metadata is missing key: usage" unless meta.key?(:usage)
+      fail InvalidMetadataError, "Metadata names is not an array" unless meta[:names].is_a?(Array)
+      fail InvalidMetadataError, "Metadata names must contain at least one string" if meta[:names].empty?
+      fail InvalidMetadataError, "Metadata usage is not string" unless meta[:usage].is_a?(String)
+    end
+  end
+end

data/lib/birdwatcher/commands/back.rb

@@ -0,0 +1,15 @@
+module Birdwatcher
+  module Commands
+    class Back < Birdwatcher::Command
+      self.meta = {
+        :description => "Unloads current module",
+        :names       => %w(back unload),
+        :usage       => "back"
+      }
+
+      def run
+        console.current_module = nil
+      end
+    end
+  end
+end

data/lib/birdwatcher/commands/exit.rb

@@ -0,0 +1,16 @@
+module Birdwatcher
+  module Commands
+    class Exit < Birdwatcher::Command
+      self.meta = {
+        :description => "Exit Birdwatcher",
+        :names       => %w(exit quit q),
+        :usage       => "exit"
+      }
+
+      def run
+        output "Goodbye."
+        exit
+      end
+    end
+  end
+end

data/lib/birdwatcher/commands/help.rb

@@ -0,0 +1,60 @@
+module Birdwatcher
+  module Commands
+    class Help < Birdwatcher::Command
+      self.meta = {
+        :description => "Show help and detailed command usage",
+        :names       => %w(help ?),
+        :usage       => "help [COMMAND]"
+      }
+
+      def self.detailed_usage
+<<-USAGE
+The #{'help'.bold} command shows a general overview of available commands as well as help
+and detailed usage for specific commands.
+
+#{'USAGE:'.bold}
+
+#{'See available commands and short descriptions:'.bold}
+  help
+
+#{'See help and detailed usage for specific command:'.bold}
+  help COMMAND
+USAGE
+      end
+
+      def run
+        if arguments?
+          show_command_help
+        else
+          show_general_help
+        end
+      end
+
+      private
+
+      def show_command_help
+        command_name = arguments.first.downcase
+        commands.each do |command|
+          next unless command.has_name?(command_name)
+          if command.detailed_usage
+            newline
+            output command.detailed_usage
+          else
+            info("There is no detailed usage for this command")
+          end
+          return
+        end
+        error "Unknown command: #{command_name}"
+      end
+
+      def show_general_help
+        longest_command_usage = commands.map { |c| c.meta[:usage] }.max_by(&:length)
+        info "Available commands:\n"
+        commands.sort_by { |c| c.meta[:usage] }.each do |command|
+          output_formatted("    %-#{longest_command_usage.bold.length}s\t\t%s\n", command.meta[:usage].bold, command.meta[:description])
+        end
+        newline
+      end
+    end
+  end
+end

data/lib/birdwatcher/commands/irb.rb

@@ -0,0 +1,34 @@
+module Birdwatcher
+  module Commands
+    class Irb < Birdwatcher::Command
+      self.meta = {
+        :description => "Start an interactive Ruby shell",
+        :names       => %w(irb),
+        :usage       => "irb"
+      }
+
+     def self.detailed_usage
+<<-USAGE
+The #{'irb'.bold} command can be used start an interactive Ruby shell (IRB) with
+all of the Birdwatcher classes and models loaded.
+
+#{'NOTE:'.bold} This command is not intended for normal users of Birdwatcher but
+can be convenient for debugging or more complex one-off data manipulation, if you
+know what you're doing.
+
+#{'USAGE:'.bold}
+
+#{'Start an interactive Ruby shell:'.bold}
+  irb
+USAGE
+      end
+
+      def run
+        require "irb"
+        require "awesome_print"
+        AwesomePrint.irb!
+        suppress_warnings { IRB.start }
+      end
+    end
+  end
+end

data/lib/birdwatcher/commands/module.rb

@@ -0,0 +1,106 @@
+module Birdwatcher
+  module Commands
+    class Module < Birdwatcher::Command
+      self.meta = {
+        :description => "Show modules",
+        :names       => %w(module modules),
+        :usage       => "module ACTION"
+      }
+
+      def self.detailed_usage
+<<-USAGE
+The #{'modules'.bold} command shows information about available Birdwatcher modules.
+
+#{'USAGE:'.bold}
+
+#{'See available modules and short descriptions:'.bold}
+  modules list
+
+#{'See detailed information on specific module:'.bold}
+  modules info MODULE_PATH
+
+#{'Search for modules with the word "import" in their name, description or path:'.bold}
+  modules search import
+
+#{'List all modules related to users:'.bold}
+  modules search users/
+USAGE
+      end
+
+      def run
+        if !arguments?
+          show_modules
+          return
+        end
+        action = arguments.first.downcase
+        case action
+        when "show", "info", "view"
+          show_module(arguments[1])
+        when "list", "-l"
+          show_modules
+        when "search", "-s"
+          search_modules
+        else
+          show_module(arguments.first)
+        end
+      end
+
+      private
+
+      def show_module(path)
+        if !_module = Birdwatcher::Module.module_by_path(path)
+          error("Unknown module: #{arguments[1].bold}")
+          return false
+        end
+        newline
+        output "       Name: ".bold + _module.meta[:name]
+        output "Description: ".bold + _module.meta[:description]
+        output "     Author: ".bold + _module.meta[:author]
+        output "       Path: ".bold + _module.path
+        newline
+        line_separator
+        newline
+        if _module.info
+          output _module.info
+        else
+          info("No further information has been provided for this module")
+        end
+        newline
+      end
+
+      def show_modules
+        info("Available Modules:\n")
+        Birdwatcher::Module.descendants.sort_by(&:path).each do |_module|
+          output_module_summary(_module)
+        end
+      end
+
+      def search_modules
+        search_term = arguments[1..-1].join(" ").downcase
+        modules     = []
+        Birdwatcher::Module.descendants.sort_by(&:path).each do |_module|
+          if _module.path.include?(search_term) || _module.meta[:name].downcase.include?(search_term) || _module.meta[:description].downcase.include?(search_term)
+            modules << _module
+          end
+        end
+        if modules.empty?
+          info("No modules found with search: #{search_term.bold}")
+        else
+          info("Module Search Results:\n")
+          modules.each do |_module|
+            output_module_summary(_module)
+          end
+        end
+      end
+
+      def output_module_summary(_module)
+        output "       Name: ".bold + _module.meta[:name]
+        output "Description: ".bold + _module.meta[:description]
+        output "       Path: ".bold + _module.path
+        newline
+        line_separator
+        newline
+      end
+    end
+  end
+end

data/lib/birdwatcher/commands/query.rb

@@ -0,0 +1,58 @@
+module Birdwatcher
+  module Commands
+    class Query < Birdwatcher::Command
+      self.meta = {
+        :description => "Execute SQL query",
+        :names       => %w(query sql),
+        :usage       => "query QUERY"
+      }
+
+      def self.detailed_usage
+<<-USAGE
+The #{'query'.bold} command can be used to execute raw SQL queries against the
+underlying database for Birdwatcher. The query results will be shown in a formatted
+table.
+
+#{'IMPORTANT:'.bold} The query command does not automatically isolate the data
+to the current workspace so queries will need to handle that on their own.
+Most tables will have a column called #{'workspace_id'.bold} which will contain
+the numeric ID of the workspace the object belongs to.
+
+For a more machine-parsable query result, please see the #{'query_csv'.bold} command.
+
+#{'USAGE EXAMPLES:'.bold}
+
+#{'See current workspaces:'.bold}
+  query SELECT * from workspaces ORDER BY name
+
+#{'See geo coordinates for all statuses in a workspace:'.bold}
+  query SELECT longitude,latitude FROM statuses WHERE geo IS TRUE AND workspace_id = 1
+
+#{'See statuses containing the word "password":'.bold}
+  query SELECT u.screen_name, s.text, s.posted_at FROM users AS u JOIN statuses AS s ON s.user_id = u.id WHERE s.text LIKE '%password%'
+
+#{'See status geographic places by frequency:'.bold}
+  query SELECT COUNT(*) AS count, place_name FROM statuses WHERE place_name IS NOT NULL GROUP BY place_name ORDER BY count DESC
+USAGE
+      end
+
+      def run
+        if !arguments?
+          error("You must provide an SQL query to execute")
+          return false
+        end
+
+        query  = arguments.join(" ")
+        result = database[query]
+        rows   = result.map { |r| r.to_hash.values }
+        table  = Terminal::Table.new(
+          :headings => result.columns.map { |c| c.to_s.bold },
+          :rows     => rows
+        ).to_s
+        page_text(table)
+      rescue Sequel::DatabaseError => e
+        error("Syntax error: #{e.message}")
+      end
+    end
+  end
+end

data/lib/birdwatcher/commands/query_csv.rb

@@ -0,0 +1,56 @@
+module Birdwatcher
+  module Commands
+    class QueryCsv < Birdwatcher::Command
+      self.meta = {
+        :description => "Execute SQL query and return result as CSV",
+        :names       => %w(query_csv csv),
+        :usage       => "query_csv QUERY"
+      }
+
+      def self.detailed_usage
+<<-USAGE
+The #{'query_csv'.bold} command can be used to execute raw SQL queries against the
+underlying database for Birdwatcher. The query results will be shown in CSV format
+for easy parsing by other tools or code.
+
+#{'IMPORTANT:'.bold} The query_csv command does not automatically isolate the data
+to the current workspace so queries will need to handle that on their own.
+Most tables will have a column called #{'workspace_id'.bold} which will contain
+the numeric ID of the workspace the object belongs to.
+
+#{'USAGE EXAMPLES:'.bold}
+
+#{'See current workspaces:'.bold}
+  query_csv SELECT * from workspaces ORDER BY name
+
+#{'See geo coordinates for all statuses in a workspace:'.bold}
+  query_csv SELECT longitude,latitude FROM statuses WHERE geo IS TRUE AND workspace_id = 1
+
+#{'See statuses containing the word "password":'.bold}
+  query_csv SELECT u.screen_name, s.text, s.posted_at FROM users AS u JOIN statuses AS s ON s.user_id = u.id WHERE s.text LIKE '%password%'
+
+#{'See status geographic places by frequency:'.bold}
+  query_csv SELECT COUNT(*) AS count, place_name FROM statuses WHERE place_name IS NOT NULL GROUP BY place_name ORDER BY count DESC
+USAGE
+      end
+
+      def run
+        if !arguments?
+          error("You must provide an SQL query to execute")
+          return false
+        end
+
+        query   = arguments.join(" ")
+        result  = database[query]
+        rows    = result.map { |r| r.to_hash.values }
+        headers = result.columns.map { |c| c.to_s }
+        csv = CSV.generate(:write_headers => true, :headers => headers) do |doc|
+          rows.each { |r| doc << r }
+        end
+        page_text(csv)
+      rescue Sequel::DatabaseError => e
+        error("Syntax error: #{e.message}")
+      end
+    end
+  end
+end