rbitter 0.1.2-java

data/lib/rbitter/default/config_json.rb

@@ -0,0 +1,37 @@
+module Rbitter
+  DEFAULT_CONFIG_JSON = <<-ENDOFJSON
+{
+  "twitter": {
+    "consumer_key": "",
+    "consumer_secret": "",
+    "access_token": "",
+    "access_token_secret": ""
+  },
+  "activerecord": "sqlite3",
+  "sqlite3": {
+    "dbfile": "rbitter.sqlite"
+  },
+  "mysql2": {
+    "host": "localhost",
+    "port": 3306,
+    "dbname": "archive",
+    "username": "",
+    "password": ""
+  },
+  "media_downloader": {
+    "large_image": true,
+    "download_dir": "imgs/"
+  },
+  "xmlrpc": {
+    "enable": true,
+    "bind_host": "0.0.0.0",
+    "bind_port": 1400,
+    "auth": {
+      "username": "username",
+      "password": "password"
+    },
+    "handles": ["/path/to/handles"]
+  }
+}
+ENDOFJSON
+end

data/lib/rbitter/dlthread.rb

@@ -0,0 +1,63 @@
+# encoding: utf-8
+
+require "net/http"
+require "openssl"
+
+module Rbitter
+  class DLThread
+    def initialize(dlfolder, large_flag)
+      @dest = dlfolder
+      if not File.directory?(dlfolder)
+        warn "[dlthread] Given download location is not available for downloading."
+        warn "[dlthread] Fallback to current directory."
+        @dest = "./"
+      end
+
+      if large_flag.nil?
+        @large_image = false
+      else
+        @large_image = large_flag
+      end
+
+      @pool = Array.new
+    end
+
+    def <<(url_array)
+      download_task = Thread.new {
+        url_array.each { |url|
+          uri = URI.parse(@large_image ? url + ":large" : url) 
+          ssl = uri.scheme.downcase == 'https'
+
+          Net::HTTP.start(uri.host, uri.port, :use_ssl => ssl) { |h|
+            req = Net::HTTP::Get.new uri.request_uri
+            h.request(req) { |res|
+              case res
+              when Net::HTTPOK
+                fname = File.basename(url)
+
+                puts "[fetch] remote: #{uri.path} => local: #{fname}"
+                open(File.join(@dest, fname), "wb") { |file|
+                  res.read_body { |chunk| file.write(chunk) }
+                }
+              end
+            }
+          }
+        }
+      }
+
+      @pool.push download_task
+    end
+
+    def job_cleanup
+      until @pool.empty?
+        dlthrd = @pool.shift
+
+        if dlthrd.alive?
+          puts "[dlthread] Thread forceful cleaning up [remains: #{@pool.length}]"
+          dlthrd.terminate
+          dlthrd.join
+        end
+      end
+    end
+  end
+end

data/lib/rbitter/env.rb

@@ -0,0 +1,62 @@
+# encoding: utf-8
+
+require "json"
+
+module Rbitter
+  @@env = Hash.new
+
+  class ConfigFileError < StandardError; end
+
+  def self.[](k)
+    @@env[k]
+  end
+
+  module_function
+  def env
+    @@env
+  end
+
+  def env_reset
+    @@env.clear
+  end
+
+  def env_validate?
+    # TODO: Add validator
+    true
+  end
+
+  def config_initialize json_path=nil
+    env_reset
+
+    unless json_path.nil?
+      begin
+        open(json_path, 'r') { |file|
+          @@env = JSON.parse(file.read)
+        }
+
+        return @@env if env_validate?
+        fail StandardError, "Invalid configuration"
+      rescue => e
+        fail ConfigFileError, "Load Failure (#{json_path}): #{e.to_s}"
+      end
+    end
+
+    # Configuration default location
+    # 1. (current_dir)/config.json
+    # 2. (current_dir)/.rbitter/config.json
+    locations = ["config.json", ".rbitter/config.json"]
+    locations.collect! { |base| File.join(Dir.pwd, base) }
+
+    for location in locations
+      next unless File.file?(location)
+      open(location, 'r') { |file|
+        @@env = JSON.parse(file.read)
+      }
+      break if env_validate?
+    end
+
+    if @@env.empty?
+      fail ConfigFileError, "Can not load any configuration in [#{locations.join(', ')}]"
+    end
+  end
+end

data/lib/rbitter/libtwitter_connection_override.rb

@@ -0,0 +1,46 @@
+# encoding: utf-8
+
+require 'http/parser'
+require 'openssl'
+require 'resolv'
+
+module Twitter
+  module Streaming
+    class Connection
+      MODIFIED = true
+      attr_reader :tcp_socket_class, :ssl_socket_class
+
+      def initialize(options = {})
+        @tcp_socket_class = options.fetch(:tcp_socket_class) { TCPSocket }
+        @ssl_socket_class = options.fetch(:ssl_socket_class) { OpenSSL::SSL::SSLSocket }
+      end
+
+      def stream(request, response)
+        client_context = OpenSSL::SSL::SSLContext.new
+        client = @tcp_socket_class.new(Resolv.getaddress(request.uri.host), request.uri.port)
+        ssl_client = @ssl_socket_class.new(client, client_context)
+        ssl_client.connect
+        request.stream(ssl_client)
+
+        loop {
+          begin
+            body = ssl_client.read_nonblock(1024) # rubocop:disable AssignmentInCondition, WhileUntilModifier
+            response << body
+          rescue IO::WaitReadable
+            # The reason for setting 90 seconds as a timeout is documented on:
+            # https://dev.twitter.com/streaming/overview/connecting
+            r, w, e = IO.select([ssl_client], [], [], 90)
+            if r.nil?
+              # If timeout occurs
+              ssl_client.close
+              raise Twitter::Error::ServerError.new("Connection stalled")
+            else
+              # If socket is readable
+              retry
+            end
+          end
+        }
+      end
+    end
+  end
+end

data/lib/rbitter/records.rb

@@ -0,0 +1,121 @@
+# encoding: utf-8
+
+require "active_record"
+require "date"
+
+module Rbitter
+  class Record < ActiveRecord::Base
+  end
+end
+
+module ARSupport
+  SCHEME_VERSION = 20150504
+  SCHEME = {
+    :marker => :integer, # 0 normal, 1 begin 2 halt
+    :marker_msg => :string,
+    :userid => :integer,
+    :username => :string,
+    :tweetid => :integer,
+    :replyto => :integer,
+    :tweet => :text, # with url unpacked
+    :date => :datetime,
+    :rt_count => :integer,
+    :fav_count => :integer
+  }
+
+  module_function
+  def prepared?
+    ActiveRecord::Base.connection.table_exists?(:records)
+  end
+
+  def connect_database
+    if Rbitter['activerecord'] == 'sqlite3'
+      warn "Warning: If you enable XMLRPC access, using sqlite is not recommended."
+      warn "Warning: Random crash can happen because of concurrency."
+      
+      if RUBY_PLATFORM == 'java'
+        require "jdbc/sqlite3"
+        Jdbc::SQLite3.load_driver
+        ActiveRecord::Base.establish_connection(
+          adapter: 'jdbcsqlite3',
+          database: Rbitter['sqlite3']['dbfile'],
+          timeout: 10000) # Long timeout for slow computer
+      else
+        ActiveRecord::Base.establish_connection(
+          adapter: 'sqlite3',
+          database: Rbitter['sqlite3']['dbfile'],
+          timeout: 10000) # Long timeout for slow computer
+      end
+    elsif Rbitter['activerecord'] == 'mysql2'
+      Jdbc::MySQL.load_driver if RUBY_PLATFORM == 'java'
+      
+      ActiveRecord::Base.establish_connection(
+        adapter: (RUBY_PLATFORM == 'java' ? 'jdbcmysql' : 'mysql2'),
+        host: Rbitter['mysql2']['host'],
+        port: Rbitter['mysql2']['port'],
+        database: Rbitter['mysql2']['dbname'],
+        username: Rbitter['mysql2']['username'],
+        password: Rbitter['mysql2']['password'],
+        encoding: "utf8mb4",
+        collation: "utf8mb4_unicode_ci")
+    else
+      raise RuntimeException.new("Unknown configuration value. 'activerecord' value should be sqlite3 or mysql2.")
+    end
+  end
+
+  def update_database_scheme
+    current_version = ActiveRecord::Migrator.current_version
+    if current_version < SCHEME_VERSION
+      warn "[records] Your ActiveRecord scheme is outdated."
+      warn "[records] Migrate... #{current_version} => #{SCHEME_VERSION}"
+      ActiveRecord::Migrator.migrate(File.expand_path("../records_migrate", __FILE__), SCHEME_VERSION)
+    end
+  end
+
+  def prepare option_string=""
+    ActiveRecord::Schema.define(version: SCHEME_VERSION) {
+      # MySQL specific option_string:
+      # utf8mb4 -> supporting UTF-8 4-byte characters (i.e. Emoji)
+      create_table(:records, { :options => option_string }) do |t|
+        SCHEME.each_key { |column|
+          case SCHEME[column]
+          when :string
+            t.string column
+          when :integer
+            t.integer column, :limit => 8
+          when :datetime
+            t.datetime column
+          when :text
+            t.text column
+          else
+            puts "Unexpected column type '#{SCHEME[column]}' of #{column}"
+          end
+        }
+      end
+
+      add_index :records, :tweetid
+    }
+  end
+
+  def any_to_datestring(obj)
+    if obj.is_a?(String)
+      # try to parse it
+      DateTime.parse(obj).strftime("%Y-%m-%d %H:%M:%S")
+    elsif obj.is_a?(DateTime) or obj.is_a?(Time)
+      obj.strftime("%Y-%m-%d %H:%M:%S")
+    else
+      raise ArgumentError.new("Can\'t automatically extract DateTime info")
+    end
+  end
+
+  def export_to_csv(csvfile)
+    open(csvfile, 'w') { |f|
+      f.write("marker,marker_msg,userid,username,tweetid,replyto,tweet,date,rt_count,fav_count")
+      f.write("\n")
+      Rbitter::Record.find_each { |t|
+        f.write("#{t.marker},#{t.marker_msg},#{t.userid},#{t.username},#{t.tweetid},")
+        f.write("#{t.replyto},#{t.tweet},#{t.date},#{t.rt_count},#{t.fav_count}\n")
+      }
+    }
+  end
+end

data/lib/rbitter/records_migrate/20150327_add_index.rb

@@ -0,0 +1,12 @@
+# encoding: utf-8
+
+
+class AddIndex < ActiveRecord::Migration
+  def up
+    add_index :records, :tweetid
+  end
+
+  def change
+    up
+  end
+end

data/lib/rbitter/records_migrate/20150504_add_replyto_column.rb

@@ -0,0 +1,12 @@
+# encoding: utf-8
+
+
+class AddReplytoColumn < ActiveRecord::Migration
+  def up
+    add_column :records, :replyto, :integer, :limit => 8
+  end
+
+  def change
+    up
+  end
+end

data/lib/rbitter/streaming.rb

@@ -0,0 +1,105 @@
+# encoding: utf-8
+
+require 'twitter'
+
+module Rbitter
+  class DummyStreamClient
+    def initialize(tokens); end
+
+    def run(&operation_block)
+      internal(&operation_block)
+    end
+
+    private
+    def internal(&operation_block)
+      tweets = [{
+        "tweetid" => 1,
+        "userid" => 1,
+        "replyto" => nil,
+        "tweet" => "test",
+        "rt_count" => 0,
+        "fav_count" => 0,
+        "screen_name" => "twitter",
+        "date" => "2015-01-01 12:11:10",
+        "media_urls" => ["https://pbs.twimg.com/media/CEPWFtgUgAEmbcV.png"],
+        "web_urls" => ["https://www.google.com/"]
+      }]
+
+      tweets.each { |tweet|
+        yield tweet
+      }
+    end
+  end
+
+  class StreamClient
+    def initialize(tokens)
+      @t = Twitter::Streaming::Client.new do |object|
+        object.consumer_key        = tokens['consumer_key']
+        object.consumer_secret     = tokens['consumer_secret']
+        object.access_token        = tokens['access_token']
+        object.access_token_secret = tokens['access_token_secret']
+      end
+    end
+
+    def run(&operation_block)
+      begin
+        internal(&operation_block)
+      rescue EOFError => e
+        puts "Network unreachable. Retry in 3 seconds..."
+        sleep 3
+        retry
+      end
+    end
+
+    private
+    def internal(&operation_block)
+      @t.user do |tweet|
+        if tweet.is_a?(Twitter::Tweet)
+          if tweet.retweet?
+            tweet = tweet.retweeted_tweet
+          end
+
+          text = tweet.full_text.gsub(/(\r\n|\n)/, '')
+
+          # unpack uris and media links
+          media_urls = Array.new
+          web_urls = Array.new
+
+          if tweet.entities?
+            if tweet.media?
+              tweet.media.each { |media|
+                media_urls.push("#{media.media_uri_https}")
+                text.gsub!("#{media.url}", "#{media.display_url}")
+              }
+            end
+
+            text += " "
+            text += media_urls.join(" ")
+
+            if tweet.uris?
+              tweet.uris.each { |uri|
+                web_urls.push("#{uri.expanded_url}")
+                text.gsub!("#{uri.url}", "#{uri.expanded_url}")
+              }
+            end
+          end
+
+          res = {
+            "tweetid" => tweet.id,
+            "userid" => tweet.user.id,
+            "replyto" => tweet.in_reply_to_status_id? ? tweet.in_reply_to_status_id : nil,
+            "tweet" => text,
+            "rt_count" => tweet.retweet_count,
+            "fav_count" => tweet.favorite_count,
+            "screen_name" => tweet.user.screen_name,
+            "date" => tweet.created_at,
+            "media_urls" => media_urls,
+            "web_urls" => web_urls
+          }
+          
+          yield res
+        end      
+      end
+    end
+  end
+end