rbitter 0.0.8 → 0.1.0

data/lib/rbitter/records.rb

@@ -1,109 +1,121 @@
-# encoding: utf-8
-
-require "active_record"
-require "date"
-
-module Rbitter
-  class Record < ActiveRecord::Base
-  end
-end
-
-module ARSupport
-  SCHEME_VERSION = 20150327
-  SCHEME = {
-    :marker => :integer, # 0 normal, 2 cut, 1 resume
-    :marker_msg => :string, # == 0 success, == 2 w/ message
-    :userid => :integer,
-    :username => :string,
-    :tweetid => :integer,
-    :tweet => :text, # with url unpacked
-    :date => :datetime,
-    :rt_count => :integer,
-    :fav_count => :integer
-  }
-
-  module_function
-  def prepared?
-    ActiveRecord::Base.connection.table_exists?(:records)
-  end
-
-  def connect_database
-    if Rbitter.env['activerecord'] == 'sqlite3'
-      warn "Warning: If you enable XMLRPC access, using sqlite is not recommended."
-      warn "Warning: Random crash can happen because of concurrency."
-      
-      if RUBY_PLATFORM == 'java'
-        require "jdbc/sqlite3"
-        Jdbc::SQLite3.load_driver
-        ActiveRecord::Base.establish_connection(
-          adapter: 'jdbcsqlite3',
-          database: Rbitter.env['sqlite3']['dbfile'],
-          timeout: 10000) # Long timeout for slow computer
-      else
-        ActiveRecord::Base.establish_connection(
-          adapter: 'sqlite3',
-          database: Rbitter.env['sqlite3']['dbfile'],
-          timeout: 10000) # Long timeout for slow computer
-      end
-    elsif Rbitter.env['activerecord'] == 'mysql2'
-      Jdbc::MySQL.load_driver if RUBY_PLATFORM == 'java'
-      
-      ActiveRecord::Base.establish_connection(
-        adapter: (RUBY_PLATFORM == 'java' ? 'jdbcmysql' : 'mysql2'),
-        host: Rbitter.env['mysql2']['host'],
-        port: Rbitter.env['mysql2']['port'],
-        database: Rbitter.env['mysql2']['dbname'],
-        username: Rbitter.env['mysql2']['username'],
-        password: Rbitter.env['mysql2']['password'],
-        encoding: "utf8mb4",
-        collation: "utf8mb4_unicode_ci")
-    else
-      raise RuntimeException.new("Unknown configuration value. 'activerecord' value should be sqlite3 or mysql2.")
-    end
-  end
-
-  def update_database_scheme
-    current_version = ActiveRecord::Migrator.current_version
-    if current_version < SCHEME_VERSION
-      warn "[records] Your ActiveRecord scheme is outdated."
-      warn "[records] Migrate... #{current_version} => #{SCHEME_VERSION}"
-      ActiveRecord::Migrator.migrate(File.expand_path("../records_migrate", __FILE__), SCHEME_VERSION)
-    end
-  end
-
-  def prepare option_string=""
-    ActiveRecord::Schema.define(version: SCHEME_VERSION) {
-      # MySQL specific option_string:
-      # utf8mb4 -> supporting UTF-8 4-byte characters (i.e. Emoji)
-      create_table(:records, { :options => option_string }) do |t|
-        SCHEME.each_key { |column|
-          case SCHEME[column]
-          when :string
-            t.string column
-          when :integer
-            t.integer column, :limit => 8
-          when :datetime
-            t.datetime column
-          when :text
-            t.text column
-          else
-            puts "Unexpected column type '#{SCHEME[column]}' of #{column}"
-          end
-        }
-      end
-
-      add_index :records, :tweetid
-    }
-  end
-
-  def any_to_datestring(obj)
-    if obj.is_a?(String)
-      # try to parse it
-      DateTime.parse(obj).strftime("%Y-%m-%d %H:%M:%S")
-    elsif obj.is_a?(DateTime) or obj.is_a?(Time)
-      obj.strftime("%Y-%m-%d %H:%M:%S")
-    else
-      raise ArgumentError.new("Can\'t automatically extract DateTime info")
-    end
-  end
-end
+# encoding: utf-8
+
+require "active_record"
+require "date"
+
+module Rbitter
+  class Record < ActiveRecord::Base
+  end
+end
+
+module ARSupport
+  SCHEME_VERSION = 20150504
+  SCHEME = {
+    :marker => :integer, # 0 normal, 1 begin 2 halt
+    :marker_msg => :string,
+    :userid => :integer,
+    :username => :string,
+    :tweetid => :integer,
+    :replyto => :integer,
+    :tweet => :text, # with url unpacked
+    :date => :datetime,
+    :rt_count => :integer,
+    :fav_count => :integer
+  }
+
+  module_function
+  def prepared?
+    ActiveRecord::Base.connection.table_exists?(:records)
+  end
+
+  def connect_database
+    if Rbitter['activerecord'] == 'sqlite3'
+      warn "Warning: If you enable XMLRPC access, using sqlite is not recommended."
+      warn "Warning: Random crash can happen because of concurrency."
+      
+      if RUBY_PLATFORM == 'java'
+        require "jdbc/sqlite3"
+        Jdbc::SQLite3.load_driver
+        ActiveRecord::Base.establish_connection(
+          adapter: 'jdbcsqlite3',
+          database: Rbitter['sqlite3']['dbfile'],
+          timeout: 10000) # Long timeout for slow computer
+      else
+        ActiveRecord::Base.establish_connection(
+          adapter: 'sqlite3',
+          database: Rbitter['sqlite3']['dbfile'],
+          timeout: 10000) # Long timeout for slow computer
+      end
+    elsif Rbitter['activerecord'] == 'mysql2'
+      Jdbc::MySQL.load_driver if RUBY_PLATFORM == 'java'
+      
+      ActiveRecord::Base.establish_connection(
+        adapter: (RUBY_PLATFORM == 'java' ? 'jdbcmysql' : 'mysql2'),
+        host: Rbitter['mysql2']['host'],
+        port: Rbitter['mysql2']['port'],
+        database: Rbitter['mysql2']['dbname'],
+        username: Rbitter['mysql2']['username'],
+        password: Rbitter['mysql2']['password'],
+        encoding: "utf8mb4",
+        collation: "utf8mb4_unicode_ci")
+    else
+      raise RuntimeException.new("Unknown configuration value. 'activerecord' value should be sqlite3 or mysql2.")
+    end
+  end
+
+  def update_database_scheme
+    current_version = ActiveRecord::Migrator.current_version
+    if current_version < SCHEME_VERSION
+      warn "[records] Your ActiveRecord scheme is outdated."
+      warn "[records] Migrate... #{current_version} => #{SCHEME_VERSION}"
+      ActiveRecord::Migrator.migrate(File.expand_path("../records_migrate", __FILE__), SCHEME_VERSION)
+    end
+  end
+
+  def prepare option_string=""
+    ActiveRecord::Schema.define(version: SCHEME_VERSION) {
+      # MySQL specific option_string:
+      # utf8mb4 -> supporting UTF-8 4-byte characters (i.e. Emoji)
+      create_table(:records, { :options => option_string }) do |t|
+        SCHEME.each_key { |column|
+          case SCHEME[column]
+          when :string
+            t.string column
+          when :integer
+            t.integer column, :limit => 8
+          when :datetime
+            t.datetime column
+          when :text
+            t.text column
+          else
+            puts "Unexpected column type '#{SCHEME[column]}' of #{column}"
+          end
+        }
+      end
+
+      add_index :records, :tweetid
+    }
+  end
+
+  def any_to_datestring(obj)
+    if obj.is_a?(String)
+      # try to parse it
+      DateTime.parse(obj).strftime("%Y-%m-%d %H:%M:%S")
+    elsif obj.is_a?(DateTime) or obj.is_a?(Time)
+      obj.strftime("%Y-%m-%d %H:%M:%S")
+    else
+      raise ArgumentError.new("Can\'t automatically extract DateTime info")
+    end
+  end
+
+  def export_to_csv(csvfile)
+    open(csvfile, 'w') { |f|
+      f.write("marker,marker_msg,userid,username,tweetid,replyto,tweet,date,rt_count,fav_count")
+      f.write("\n")
+      Rbitter::Record.find_each { |t|
+        f.write("#{t.marker},#{t.marker_msg},#{t.userid},#{t.username},#{t.tweetid},")
+        f.write("#{t.replyto},#{t.tweet},#{t.date},#{t.rt_count},#{t.fav_count}\n")
+      }
+    }
+  end
+end

data/lib/rbitter/records_migrate/20150327_add_index.rb

@@ -1,8 +1,12 @@
-# encoding: utf-8
-
-
-class AddIndex < ActiveRecord::Migration
-  def self.change
-    add_index :records, :tweetid
-  end
+# encoding: utf-8
+
+
+class AddIndex < ActiveRecord::Migration
+  def up
+    add_index :records, :tweetid
+  end
+
+  def change
+    up
+  end
 end

data/lib/rbitter/records_migrate/20150504_add_replyto_column.rb

@@ -0,0 +1,12 @@
+# encoding: utf-8
+
+
+class AddReplytoColumn < ActiveRecord::Migration
+  def up
+    add_column :records, :replyto, :integer, :limit => 8
+  end
+
+  def change
+    up
+  end
+end

data/lib/rbitter/streaming.rb

@@ -1,76 +1,105 @@
-# encoding: utf-8
-
-require 'twitter'
-
-module Rbitter
-  class StreamClient
-    def initialize(tokens)
-      @t = Twitter::Streaming::Client.new do |object|
-        object.consumer_key        = tokens['consumer_key']
-        object.consumer_secret     = tokens['consumer_secret']
-        object.access_token        = tokens['access_token']
-        object.access_token_secret = tokens['access_token_secret']
-      end
-    end
-
-    def run(&operation_block)
-      begin
-        internal(&operation_block)
-      rescue EOFError => e
-        puts "Network unreachable. Retry in 3 seconds..."
-        sleep 3
-        retry
-      end
-    end
-
-    private
-    def internal(&operation_block)
-      @t.user do |tweet|
-        if tweet.is_a?(Twitter::Tweet)
-          if tweet.retweet?
-            tweet = tweet.retweeted_tweet
-          end
-
-          text = tweet.full_text.gsub(/(\r\n|\n)/, '')
-
-          # unpack uris and media links
-          media_urls = Array.new
-          web_urls = Array.new
-
-          if tweet.entities?
-            if tweet.media?
-              tweet.media.each { |media|
-                media_urls.push("#{media.media_uri_https}")
-                text.gsub!("#{media.url}", "#{media.display_url}")
-              }
-            end
-
-            text += " "
-            text += media_urls.join(" ")
-
-            if tweet.uris?
-              tweet.uris.each { |uri|
-                web_urls.push("#{uri.expanded_url}")
-                text.gsub!("#{uri.url}", "#{uri.expanded_url}")
-              }
-            end
-          end
-
-          res = {
-            "tweetid" => tweet.id,
-            "userid" => tweet.user.id,
-            "tweet" => text,
-            "rt_count" => tweet.retweet_count,
-            "fav_count" => tweet.favorite_count,
-            "screen_name" => tweet.user.screen_name,
-            "date" => tweet.created_at,
-            "media_urls" => media_urls,
-            "web_urls" => web_urls
-          }
-          
-          yield res
-        end      
-      end
-    end
-  end
+# encoding: utf-8
+
+require 'twitter'
+
+module Rbitter
+  class DummyStreamClient
+    def initialize(tokens); end
+
+    def run(&operation_block)
+      internal(&operation_block)
+    end
+
+    private
+    def internal(&operation_block)
+      tweets = [{
+        "tweetid" => 1,
+        "userid" => 1,
+        "replyto" => nil,
+        "tweet" => "test",
+        "rt_count" => 0,
+        "fav_count" => 0,
+        "screen_name" => "twitter",
+        "date" => "2015-01-01 12:11:10",
+        "media_urls" => ["https://pbs.twimg.com/media/CEPWFtgUgAEmbcV.png"],
+        "web_urls" => ["https://www.google.com/"]
+      }]
+
+      tweets.each { |tweet|
+        yield tweet
+      }
+    end
+  end
+
+  class StreamClient
+    def initialize(tokens)
+      @t = Twitter::Streaming::Client.new do |object|
+        object.consumer_key        = tokens['consumer_key']
+        object.consumer_secret     = tokens['consumer_secret']
+        object.access_token        = tokens['access_token']
+        object.access_token_secret = tokens['access_token_secret']
+      end
+    end
+
+    def run(&operation_block)
+      begin
+        internal(&operation_block)
+      rescue EOFError => e
+        puts "Network unreachable. Retry in 3 seconds..."
+        sleep 3
+        retry
+      end
+    end
+
+    private
+    def internal(&operation_block)
+      @t.user do |tweet|
+        if tweet.is_a?(Twitter::Tweet)
+          if tweet.retweet?
+            tweet = tweet.retweeted_tweet
+          end
+
+          text = tweet.full_text.gsub(/(\r\n|\n)/, '')
+
+          # unpack uris and media links
+          media_urls = Array.new
+          web_urls = Array.new
+
+          if tweet.entities?
+            if tweet.media?
+              tweet.media.each { |media|
+                media_urls.push("#{media.media_uri_https}")
+                text.gsub!("#{media.url}", "#{media.display_url}")
+              }
+            end
+
+            text += " "
+            text += media_urls.join(" ")
+
+            if tweet.uris?
+              tweet.uris.each { |uri|
+                web_urls.push("#{uri.expanded_url}")
+                text.gsub!("#{uri.url}", "#{uri.expanded_url}")
+              }
+            end
+          end
+
+          res = {
+            "tweetid" => tweet.id,
+            "userid" => tweet.user.id,
+            "replyto" => tweet.in_reply_to_status_id? ? tweet.in_reply_to_status_id : nil,
+            "tweet" => text,
+            "rt_count" => tweet.retweet_count,
+            "fav_count" => tweet.favorite_count,
+            "screen_name" => tweet.user.screen_name,
+            "date" => tweet.created_at,
+            "media_urls" => media_urls,
+            "web_urls" => web_urls
+          }
+          
+          yield res
+        end      
+      end
+    end
+  end
 end

data/lib/rbitter/version.rb

@@ -1,3 +1,20 @@
-module Rbitter
-  VERSION = "0.0.8"
-end
+module Rbitter
+  PRODUCT_NAME = "Rbitter"
+  VERSION = "0.1.0"
+
+  def major
+    VERSION.match(/^([0-9]+)\./)[1]
+  end
+
+  def minor
+    VERSION.match(/\.([0-9]+)\./)[1]
+  end
+
+  def patchlv
+    VERSION.match(/\.([0-9]+)$/)[1]
+  end
+
+  def version_string
+    "#{PRODUCT_NAME} #{VERSION}"
+  end
+end

data/lib/rbitter/xmlrpc.rb

@@ -1,4 +1,4 @@
-# encoding: utf-8
-#
-
+# encoding: utf-8
+#
+
 require "rbitter/xmlrpcd/xmlrpcd"

data/lib/rbitter/xmlrpcd/base.rb

@@ -1,25 +1,25 @@
-# encoding: utf-8
-
-module RPCHandles
-  RH_INFO = Struct.new("RPCHANDLE_INFO", :name, :version, :author, :description) {
-    def digest
-      "<rpchandle: #{name}-#{version} (written by #{author}, #{description})>"
-    end
-  }
-
-  module BaseHandle
-    # If a handler doesn't require an authorization, please inherit below class
-    class NoAuth < Object
-      def self.auth?
-        false
-      end
-    end
-
-    # If a handler does require an authorization, please inherit below class
-    class Auth < Object
-      def self.auth?
-        true
-      end
-    end
-  end
+# encoding: utf-8
+
+module RPCHandles
+  RH_INFO = Struct.new("RPCHANDLE_INFO", :name, :version, :author, :description) {
+    def digest
+      "<rpchandle: #{name}-#{version} (written by #{author}, #{description})>"
+    end
+  }
+
+  module BaseHandle
+    # If a handler doesn't require an authorization, please inherit below class
+    class NoAuth < Object
+      def self.auth?
+        false
+      end
+    end
+
+    # If a handler does require an authorization, please inherit below class
+    class Auth < Object
+      def self.auth?
+        true
+      end
+    end
+  end
 end

data/lib/rbitter/xmlrpcd/rpchandles.rb

@@ -1,12 +1,12 @@
-# encoding: utf-8
-
-module RPCHandles
-  # Override this function will activate authentication feature.
-  # You can write and add RPCHandle. See 'rpc' folder.
-
-  @@auth_pool = nil
-  module_function
-  def auth
-    @@auth_pool
-  end
+# encoding: utf-8
+
+module RPCHandles
+  # Override this function will activate authentication feature.
+  # You can write and add RPCHandle. See 'rpc' folder.
+
+  @@auth_pool = nil
+  module_function
+  def auth
+    @@auth_pool
+  end
 end