rbitter 0.0.8

data/lib/rbitter/records_migrate/20150327_add_index.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+
+
+class AddIndex < ActiveRecord::Migration
+  def self.change
+    add_index :records, :tweetid
+  end
+end

data/lib/rbitter/streaming.rb

@@ -0,0 +1,76 @@
+# encoding: utf-8
+
+require 'twitter'
+
+module Rbitter
+  class StreamClient
+    def initialize(tokens)
+      @t = Twitter::Streaming::Client.new do |object|
+        object.consumer_key        = tokens['consumer_key']
+        object.consumer_secret     = tokens['consumer_secret']
+        object.access_token        = tokens['access_token']
+        object.access_token_secret = tokens['access_token_secret']
+      end
+    end
+
+    def run(&operation_block)
+      begin
+        internal(&operation_block)
+      rescue EOFError => e
+        puts "Network unreachable. Retry in 3 seconds..."
+        sleep 3
+        retry
+      end
+    end
+
+    private
+    def internal(&operation_block)
+      @t.user do |tweet|
+        if tweet.is_a?(Twitter::Tweet)
+          if tweet.retweet?
+            tweet = tweet.retweeted_tweet
+          end
+
+          text = tweet.full_text.gsub(/(\r\n|\n)/, '')
+
+          # unpack uris and media links
+          media_urls = Array.new
+          web_urls = Array.new
+
+          if tweet.entities?
+            if tweet.media?
+              tweet.media.each { |media|
+                media_urls.push("#{media.media_uri_https}")
+                text.gsub!("#{media.url}", "#{media.display_url}")
+              }
+            end
+
+            text += " "
+            text += media_urls.join(" ")
+
+            if tweet.uris?
+              tweet.uris.each { |uri|
+                web_urls.push("#{uri.expanded_url}")
+                text.gsub!("#{uri.url}", "#{uri.expanded_url}")
+              }
+            end
+          end
+
+          res = {
+            "tweetid" => tweet.id,
+            "userid" => tweet.user.id,
+            "tweet" => text,
+            "rt_count" => tweet.retweet_count,
+            "fav_count" => tweet.favorite_count,
+            "screen_name" => tweet.user.screen_name,
+            "date" => tweet.created_at,
+            "media_urls" => media_urls,
+            "web_urls" => web_urls
+          }
+          
+          yield res
+        end      
+      end
+    end
+  end
+end

data/lib/rbitter/version.rb

@@ -0,0 +1,3 @@
+module Rbitter
+  VERSION = "0.0.8"
+end

data/lib/rbitter/xmlrpc.rb

@@ -0,0 +1,4 @@
+# encoding: utf-8
+#
+
+require "rbitter/xmlrpcd/xmlrpcd"

data/lib/rbitter/xmlrpcd/base.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+
+module RPCHandles
+  RH_INFO = Struct.new("RPCHANDLE_INFO", :name, :version, :author, :description) {
+    def digest
+      "<rpchandle: #{name}-#{version} (written by #{author}, #{description})>"
+    end
+  }
+
+  module BaseHandle
+    # If a handler doesn't require an authorization, please inherit below class
+    class NoAuth < Object
+      def self.auth?
+        false
+      end
+    end
+
+    # If a handler does require an authorization, please inherit below class
+    class Auth < Object
+      def self.auth?
+        true
+      end
+    end
+  end
+end

data/lib/rbitter/xmlrpcd/rpchandles.rb

@@ -0,0 +1,12 @@
+# encoding: utf-8
+
+module RPCHandles
+  # Override this function will activate authentication feature.
+  # You can write and add RPCHandle. See 'rpc' folder.
+
+  @@auth_pool = nil
+  module_function
+  def auth
+    @@auth_pool
+  end
+end

data/lib/rbitter/xmlrpcd/xmlrpc_auth_server.rb

@@ -0,0 +1,84 @@
+# encoding: utf-8
+#
+
+require "rbitter/xmlrpcd/rpchandles"
+require "rbitter/xmlrpcd/base"
+require "xmlrpc/server"
+require "webrick"
+
+module XMLRPC
+  class HTTPAuthXMLRPCServer < XMLRPC::WEBrickServlet
+    def extract_method(methodname, *args)
+      for name, obj in @handler
+        if obj.kind_of? Proc
+          next unless methodname == name
+        else
+          next unless methodname =~ /^#{name}(.+)$/
+          next unless obj.respond_to? $1
+          return obj.method($1)
+        end
+      end
+      nil
+    end
+
+    def service(request, response)
+      # Taken from xmlrpc/server.rb
+      if @valid_ip
+        raise WEBrick::HTTPStatus::Forbidden unless @valid_ip.any? { |ip| request.peeraddr[3] =~ ip }
+      end
+
+      if request.request_method != "POST"
+        raise WEBrick::HTTPStatus::MethodNotAllowed,
+              "unsupported method `#{request.request_method}'."
+      end
+
+      if parse_content_type(request['Content-type']).first != "text/xml"
+        raise WEBrick::HTTPStatus::BadRequest
+      end
+
+      length = (request['Content-length'] || 0).to_i
+
+      raise WEBrick::HTTPStatus::LengthRequired unless length > 0
+
+      data = request.body
+
+      if data.nil? or data.bytesize != length
+        raise WEBrick::HTTPStatus::BadRequest
+      end
+
+      # Originally, process(data) was here.
+      # We need to check whether a method requires authorization.
+      rpc_method_name, rpc_params = parser().parseMethodCall(data)
+      rpc_method = extract_method(rpc_method_name)
+
+      if RPCHandles.auth.nil?
+        resp = handle(rpc_method_name, *rpc_params)
+      else
+        if rpc_method.owner.ancestors.include?(RPCHandles::BaseHandle::Auth)
+          # Check cookie and check it's valid
+          if request.cookies.size == 1 \
+            and request.cookies[0].name == "auth_key" \
+            and RPCHandles.auth.include?(request.cookies[0].value)
+            resp = handle(rpc_method_name, *rpc_params)
+          else
+            # Permission required
+            raise WEBrick::HTTPStatus::Forbidden
+          end
+        elsif rpc_method.owner.ancestors.include?(RPCHandles::BaseHandle::NoAuth)
+          resp = handle(rpc_method_name, *rpc_params)
+        else
+          raise WEBrick::HTTPStatus::Forbidden
+        end
+      end
+
+      if resp.nil? or resp.bytesize <= 0
+        raise WEBrick::HTTPStatus::InternalServerError
+      end
+
+      response.status = 200
+      response['Content-Length'] = resp.bytesize
+      response['Content-Type']   = "text/xml; charset=utf-8"
+      response.body = resp
+    end
+  end
+end

data/lib/rbitter/xmlrpcd/xmlrpcd.rb

@@ -0,0 +1,63 @@
+# encoding: utf-8
+
+require "rbitter/xmlrpcd/rpchandles"
+require "rbitter/xmlrpcd/xmlrpc_auth_server"
+require "webrick"
+
+module Rbitter
+  RPC_PREFIX="rbitter"
+
+  class RPCServer
+    def initialize bind_host, bind_port
+      @server = WEBrick::HTTPServer.new(:Port => bind_port.to_i, :BindAddress => bind_host.to_s, :MaxClients => 4, :Logger => WEBrick::Log.new($stdout))
+      @core = XMLRPC::HTTPAuthXMLRPCServer.new
+      load_all_handles
+      @core.set_default_handler { |name, *args|
+        "NO_COMMAND: #{name} with args #{args.inspect}"
+      }
+    end
+
+    def load_all_handles
+      Rbitter.env["xmlrpc"]["handles"].each { |path|
+        puts "[xmlrpc] Scanning handles from (#{path})"
+        Dir.entries(path).each { |fname|
+          fname = File.join(path, fname)
+          if File.exist?(fname) and File.file?(fname)
+            if fname.match(/rh_\w+\.rb$/)
+              begin
+                load fname
+              rescue Exception => e
+                # stub
+                puts "Exception while loading #{fname}"
+                puts e.inspect
+              end
+            else
+              puts "Ignored: #{fname}"
+            end
+          end
+        }
+      }
+
+      puts "[xmlrpc] found #{RPCHandles.constants.length} constants."
+      RPCHandles.constants.each { |handler|
+        if RPCHandles.const_get(handler).is_a?(Class)
+          @core.add_handler(RPC_PREFIX, RPCHandles.const_get(handler).new)
+        end
+      }
+    end
+
+    def main_loop
+      puts "RPCServer starts"
+      @server.mount("/", @core)
+      @server.start
+    end
+  end
+end
+
+
+if __FILE__ == $0
+  puts "Local testing server starts..."
+  rpcd = Rbitter::RPCServer.new('127.0.0.1', 1300)
+  rpcd.main_loop
+end
+

data/lib/rbitter.rb

@@ -0,0 +1,92 @@
+# encoding: utf-8
+#
+
+require "rbitter/version"
+require "rbitter/arcserver"
+require "rbitter/env"
+require "rbitter/console"
+require "rbitter/xmlrpc"
+
+module Rbitter
+  def self.rbitter_header
+    puts "Rbitter #{VERSION} on Ruby-#{RUBY_VERSION} (#{RUBY_PLATFORM})"
+  end
+
+  def self.rbitter_help_msg  
+    puts "Rbitter is a Twitter streaming archiver, with XMLRPC access."
+    puts "-------"
+    puts "Usage: rbitter (application-mode)"
+    puts "application-mode's are:"
+    puts "|- serve    : Launch Rbitter full system (Streaming + Database + XMLRPC)"
+    puts "|- console  : Launch console application utilizing XMLRPC"
+    puts "|- configure: Write default configuration file 'config.json' in current folder"
+    puts "|- help     : Show this message"
+    puts "`- logs     : Show Rbitter internal logs"
+  end
+
+  def self.prebootstrap
+    # Due to stalled socket problem, If unpatched twitter gem is installed.
+    # Twitter::Streaming::Connection will be monkey-patched.
+    patch_required = false
+
+    if Twitter::Version.const_defined?(:MAJOR)
+      b5_version = Twitter::Version::MAJOR * 10000
+      + Twitter::Version::MINOR * 100 + Twitter::Version::PATCH
+      if b5_version <= 51400
+        warn "[rbitter] Monkey-patching Twitter::Streaming::Connection..."
+        warn "[rbitter] Gem installed on here seemed that it doesn't handle socket read timeout."
+        warn "[rbitter] Please upgrade twitter gem"
+        patch_required = true
+      end
+    else
+      b6_version = Twitter::Version.to_a
+      if b6_version[0] <= 6 and b6_version[1] <= 0 and b6_version[2] <= 0
+        patch_required = true
+      end
+    end
+
+    require "rbitter/libtwitter_connection_override" if patch_required
+  end
+
+  def self.bootstrap args=[]
+    prebootstrap
+
+    if args.length == 0
+      rbitter_header
+    elsif args[0] == "serve"
+      Rbitter.config_initialize
+
+      main = Rbitter::ArcServer.new
+      if env['xmlrpc']['enable']
+        $rpc_service_thread = Thread.new {
+          rpc_server = Rbitter::RPCServer.new(env['xmlrpc']['bind_host'], env['xmlrpc']['bind_port'])
+          rpc_server.main_loop
+        }
+        $rpc_service_thread.run
+      end
+      main.main_loop
+    elsif args[0] == "configure"
+      require "rbitter/default/config_json"
+
+      puts "Writing config.json now"
+      open(File.join(Dir.pwd, "config.json"), "w") { |io|
+        io.write(DEFAULT_CONFIG_JSON)
+      }
+      puts "Writing finished"
+      puts "You can put config.json one of these locations:"
+      puts "[1] config.json (current folder)"
+      puts "[2] .rbitter/config.json (current folder)"
+    elsif args[0] == "console"
+      Rbitter.config_initialize
+
+      con = Rbitter::Console.new
+      con.start
+    elsif args[0] == "logs"
+      # show log in stdout
+      puts "This feature is in heavy development. Sorry."
+    else
+      rbitter_header
+      rbitter_help_msg
+    end
+  end
+end

data/rbitter.gemspec

@@ -0,0 +1,42 @@
+# coding: utf-8
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rbitter/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "rbitter"
+  spec.version       = Rbitter::VERSION
+  spec.authors       = ["Nidev Plontra"]
+  spec.email         = ["nidev.plontra@gmail.com"]
+  spec.summary       = %q{Rbitter is a Twitter client specialized in archiving}
+  spec.description   = %q{Rbitter archives all tweets appeared on user streaming using ActiveRecord. XMLRPC is used to serve archived tweets and useful features}
+  spec.homepage      = "https://github.com/nidev/rbitter"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+
+  spec.add_dependency 'twitter', '~> 5.14'
+  spec.add_dependency 'json', '~> 1.7'
+  spec.add_dependency 'ripl', '~> 0'
+  spec.add_dependency 'activerecord', '~> 4.0'
+
+  if RUBY_PLATFORM == 'java'
+    spec.add_dependency 'activerecord-jdbc-adapter', '~> 1.3'
+    spec.add_dependency 'jdbc-sqlite3', '~> 3.8'
+    spec.add_dependency 'jdbc-mysql', '~> 5.1'
+    spec.add_dependency 'activerecord-jdbcsqlite3-adapter', '~> 1.3'
+    spec.add_dependency 'activerecord-jdbcmysql-adapter', '~> 1.3'
+  else
+    spec.add_dependency 'sqlite3', '~> 1.3'
+    spec.add_dependency 'mysql2', '~> 0.3'
+    spec.add_dependency 'activerecord-mysql2-adapter', '~> 0.0.3'
+  end
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/spec/config/default.json

@@ -0,0 +1,33 @@
+{
+  "twitter": {
+    "consumer_key": "",
+    "consumer_secret": "",
+    "access_token": "",
+    "access_token_secret": ""
+  },
+  "activerecord": "sqlite3",
+  "sqlite3": {
+    "dbfile": "rbitter.sqlite"
+  },
+  "mysql2": {
+    "host": "localhost",
+    "port": 3306,
+    "dbname": "archive",
+    "username": "",
+    "password": ""
+  },
+  "media_downloader": {
+    "cacert_path": "/cacerts/cacert.pem",
+    "download_dir": "imgs/"
+  },
+  "xmlrpc": {
+    "enable": true,
+    "bind_host": "0.0.0.0",
+    "bind_port": 1400,
+    "auth": {
+      "username": "username",
+      "password": "password"
+    },
+    "handles": ["/path/to/handles"]
+  }
+}

data/spec/rbitter/arcserver_spec.rb

@@ -0,0 +1,9 @@
+# encoding: utf-8
+require "rbitter/arcserver"
+
+describe Rbitter::ArcServer do
+  # TODO: Perform test...
+  it 'is presented' do
+    expect(Rbitter::ArcServer).to be_a(Class)
+  end
+end

data/spec/rbitter/console_spec.rb

@@ -0,0 +1,9 @@
+# encoding: utf-8
+require "rbitter/console"
+
+describe Rbitter::Console do
+  # TODO: Perform test...
+  it 'is presented' do
+    expect(Rbitter::Console).to be_a(Class)
+  end
+end

data/spec/rbitter/default/config_json_spec.rb

@@ -0,0 +1,3 @@
+# encoding: utf-8
+
+require "rbitter/default/config_json"

data/spec/rbitter/dlthread_spec.rb

@@ -0,0 +1,9 @@
+# encoding: utf-8
+require "rbitter/dlthread"
+
+describe Rbitter::DLThread do
+  # TODO: Perform test...
+  it 'is presented' do
+    expect(Rbitter::DLThread).to be_a(Class)
+  end
+end

data/spec/rbitter/env_spec.rb

@@ -0,0 +1,56 @@
+# encoding: utf-8
+
+require "rbitter"
+
+describe Rbitter do
+  it 'has env_reset function and clears Rbitter.env,' do
+    Rbitter.env_reset
+    expect(Rbitter.env).to be_a(Hash)
+    expect(Rbitter.env.length).to be(0)
+  end
+
+  context 'when config.json is not installed,' do
+    it 'fails on loading' do
+      expect{Rbitter.config_initialize}.to raise_error(Rbitter::ConfigurationFileError)
+    end
+  end
+
+  context 'when path to config.json is invalid,' do
+    it 'fals on loading' do
+      expect{Rbitter.config_initialize("/silly/dummy/.")}.to raise_error(Rbitter::ConfigurationFileError)
+    end
+  end
+
+
+  context 'when default config.json is installed,' do
+    before(:all) do
+      Rbitter.bootstrap(['configure'])
+      expect(File.file?('config.json')).to be(true)
+    end
+
+    it 'loads configuration successfully with autodetection' do
+      expect{Rbitter.config_initialize}.to_not raise_error
+      expect(Rbitter.env.length > 0).to be(true)
+    end
+
+    it 'loads configuration successfully with given config.json path' do
+      expect{Rbitter.config_initialize('config.json')}.to_not raise_error
+      expect(Rbitter.env.length > 0).to be(true)
+    end
+
+    it 'checks that Rbitter.env returns Hash' do
+      expect(Rbitter.env).to be_a(Hash)
+    end
+
+    after(:all) do
+      File.delete('config.json')
+    end
+  end
+
+  context 'when config.json is corrupted' do
+    # TODO: Perform test with spec/config/default.json
+    # TODO: Adding configuration validator on env.rb
+  end
+
+  # TODO: Perform test with spec/config/default.json
+end

data/spec/rbitter/libtwitter_connection_override_spec.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+require "rbitter/libtwitter_connection_override"
+
+describe Rbitter do
+  it 'overrides twitter gem' do
+    expect(Twitter::Streaming::Connection::MODIFIED).to be(true)
+  end
+end

data/spec/rbitter/records_spec.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+require "rbitter/records"
+
+describe Rbitter::Record do
+  # TODO: Perform test...
+  it 'has ActiveRecord class named Record' do
+    expect(Rbitter::Record).to be_a(Class)
+  end
+
+  it 'has supportive module of ActiveRecord' do
+    expect(ARSupport).to be_a(Module)
+  end
+end

data/spec/rbitter/streaming_spec.rb

@@ -0,0 +1,9 @@
+# encoding: utf-8
+require "rbitter/streaming"
+
+describe Rbitter::StreamClient do
+  # TODO: Perform test...
+  it 'is presented' do
+    expect(Rbitter::StreamClient).to be_a(Class)
+  end
+end

data/spec/rbitter/version_spec.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+require "rbitter/version"
+
+describe Rbitter do
+  it 'has version string' do
+    expect(Rbitter::VERSION).to be_a(String)
+  end
+end

data/spec/rbitter/xmlrpc_spec.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+require "rbitter/xmlrpc"
+
+describe Rbitter do
+  it 'Rbitter is a module' do
+    expect(Rbitter.class === Module).to be(true)
+  end
+end

data/spec/rbitter/xmlrpcd/base_spec.rb

@@ -0,0 +1,29 @@
+# encoding: utf-8
+
+require "rbitter/xmlrpcd/base"
+
+describe RPCHandles do
+  context 'when RH_INFO Struct is served as RPCHandle information,' do
+    it 'structs corrct Struct::RPCHANDLE_INFO' do
+      # String comparison: use eq(==), not be(===)
+      expect(RPCHandles::RH_INFO.to_s).to eq("Struct::RPCHANDLE_INFO")
+    end
+
+    it 'provides correct information when #digest is called' do
+      digest_string = RPCHandles::RH_INFO.new('test', 0.1, 'test', 'test').digest
+      expect(digest_string).to eq("<rpchandle: test-0.1 (written by test, test)>")
+    end
+  end
+end
+
+describe RPCHandles::BaseHandle do
+  context 'when an RPCHandle inherits one of classes Auth and NoAuth' do
+    it 'returns false on #auth? when an RPCHandle class inherits NoAuth' do
+      expect(RPCHandles::BaseHandle::NoAuth.auth?).to be(false)
+    end
+
+    it 'returns true on #auth? when an RPCHandle class inherits Auth' do
+      expect(RPCHandles::BaseHandle::Auth.auth?).to be(true)
+    end
+  end
+end

data/spec/rbitter/xmlrpcd/rpchandles_spec.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+require "rbitter/xmlrpcd/rpchandles"
+
+describe RPCHandles do
+  context 'When features related to authentication needs ::auth method,'
+  it 'responds to ::auth' do
+    expect(RPCHandles.respond_to?(:auth)).to be(true)
+    expect(RPCHandles.auth).to be_a(NilClass)
+  end
+end

data/spec/rbitter/xmlrpcd/xmlrpc_auth_server_spec.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+require "rbitter/xmlrpcd/xmlrpc_auth_server"
+
+describe Rbitter do
+  it 'Rbitter is a module' do
+    expect(Rbitter.class === Module).to be(true)
+  end
+end

data/spec/rbitter/xmlrpcd/xmlrpcd_spec.rb

@@ -0,0 +1,9 @@
+# encoding: utf-8
+
+require "rbitter/xmlrpcd/xmlrpcd"
+
+describe Rbitter do
+  it 'Rbitter is a module' do
+    expect(Rbitter.class === Module).to be(true)
+  end
+end