rbitter 0.0.8 → 0.1.0

data/lib/rbitter/console.rb

@@ -1,61 +1,93 @@
-# encoding: utf-8
-#
-# Rbitter Archive Access console (irb)
-
-require "xmlrpc/client"
-require "ripl"
-
-module Rbitter
-  class Console
-    def initialize
-      puts "Welcome to Rbitter console"
-      help
-    end
-
-    def help
-      puts "Predefined methods:"
-      puts "help - to show this message again"
-      puts "xmlrpc - to utilize xmlrpc"
-      puts "activerec - to connect and utilize Rbitter::Record"
-      puts "^D, 'exit' - to get out from here."
-    end
-
-    def activerec
-      ARSupport.connect_database
-      puts "Rbitter::Record is ready."
-    end
-
-    def exit
-      Kernel.exit(0)
-    end
-    
-    def xmlrpc *args
-      if args.empty?
-        puts "How to use: xmlrpc (command) [params in Array]"
-        puts "Ex) xmlrpc rbitter.echo [\"Hello World!\"]"
-        puts "Ex) To call XMLRPC function with zero parameter, use nil."
-        return false
-      end
-
-      cl = XMLRPC::Client.new('localhost', '/', 1400) # TODO: External address?
-      if @xmlrpc_cookie.nil?
-        @xmlrpc_cookie = "auth_key=" + cl.call('rbitter.auth', Rbitter.env['xmlrpc']['auth'][0], Rbitter.env['xmlrpc']['auth'][1])
-      end
-      
-      if cl.cookie != "auth_key="
-        if args.length <= 1 or args[1].nil?
-          cl.call(args[0])
-        else
-          cl.call(args[0], *args[1])
-        end
-      else
-        puts "Authentication failed. Check your config.json"
-      end
-    end
-
-    def start
-      Ripl.start :binding => binding
-    end
-  end
-end
-
+# encoding: utf-8
+#
+# Rbitter Archive Access console (irb)
+
+require "xmlrpc/client"
+require "rbitter/version"
+require "ripl"
+
+module Rbitter
+  class Console
+    def initialize
+      puts "Rbitter console #{Rbitter::VERSION}"
+      help
+    end
+
+    def help
+      puts "Predefined methods:"
+      puts "ar - shortcut to call Rbitter::Record"
+      puts "connect_ar - Prepare Rbitter::Record to be ready"
+      puts "csv_backup - export Rbitter::Record into comma-separated values"
+      puts "help - to show this message again"
+      puts "xmlrpc - send xmlrpc command to destination"
+      puts "xmlrpc_dest - set destination for xmlrpc command"
+      puts "^D, 'exit' to exit from here"
+    end
+
+    def connect_ar
+      ARSupport.connect_database
+      puts "Rbitter::Record is ready."
+    end
+
+    def csv_backup *args
+      if args.length < 0
+        puts "Usage: csv_backup('filename.csv')"
+        puts "Estimated running time depends on system environment"
+      else
+        ARSupport.export_to_csv(args[0])
+      end
+    end
+
+    def ar
+      Rbitter::Record
+    end
+
+    def exit
+      Kernel.exit(0)
+    end
+    
+    def xmlrpc_dest args={}
+      if args.empty?
+        puts "Usage: xmlrpc_dest({ :rpchost => '', :rpcpath => '', :rpcport => 1400,"
+        puts "                   :xmlrpc_auth_id => '', xmlrpc_auth_password => '' })"
+      end
+
+      @rpchost = args.fetch(:rpchost) { "127.0.0.1" }
+      @rpcpath = args.fetch(:rpcpath) { "/" }
+      @rpcport = args.fetch(:rpcport) { 1400 }
+
+      cl = XMLRPC::Client.new(@rpchost, @rpcpath, @rpcport)
+      @xmlrpc_cookie = "auth_key=" + cl.call('rbitter.auth',
+        args.fetch(:xmlrpc_auth_id) { Rbitter.env['xmlrpc']['auth'][0] },
+        args.fetch(:xmlrpc_auth_password) { Rbitter.env['xmlrpc']['auth'][1] } )
+
+      if @xmlrpc_cookie != "auth_key="
+        puts "Authentication completed"
+      else
+        puts "Authentication failed"
+      end
+    end
+
+    def xmlrpc *args
+      if args.empty?
+        puts "Usage: xmlrpc(command, [params in Array])"
+        puts "Ex) xmlrpc(\'rbitter.echo\',' [\"Hello World!\"])"
+        puts "Please configure XMLRPC destination with xmlrpc_dest method"
+        return false
+      end
+
+      cl = XMLRPC::Client.new(@rpchost, @rpcpath, @rpcport, @xmlrpc_cookie)
+      
+      if args.length <= 1 or args[1].nil?
+        cl.call(args[0])
+      else
+        cl.call(args[0], *args[1])
+      end
+    end
+
+    def start
+      Ripl.start :binding => binding
+    end
+  end
+end
+

data/lib/rbitter/default/config_json.rb

@@ -1,37 +1,38 @@
-module Rbitter
-  DEFAULT_CONFIG_JSON = <<-ENDOFJSON
-{
-  "twitter": {
-    "consumer_key": "",
-    "consumer_secret": "",
-    "access_token": "",
-    "access_token_secret": ""
-  },
-  "activerecord": "sqlite3",
-  "sqlite3": {
-    "dbfile": "rbitter.sqlite"
-  },
-  "mysql2": {
-    "host": "localhost",
-    "port": 3306,
-    "dbname": "archive",
-    "username": "",
-    "password": ""
-  },
-  "media_downloader": {
-    "cacert_path": "/cacerts/cacert.pem",
-    "download_dir": "imgs/"
-  },
-  "xmlrpc": {
-    "enable": true,
-    "bind_host": "0.0.0.0",
-    "bind_port": 1400,
-    "auth": {
-      "username": "username",
-      "password": "password"
-    },
-    "handles": ["/path/to/handles"]
-  }
-}
-ENDOFJSON
-end
+module Rbitter
+  DEFAULT_CONFIG_JSON = <<-ENDOFJSON
+{
+  "twitter": {
+    "consumer_key": "",
+    "consumer_secret": "",
+    "access_token": "",
+    "access_token_secret": ""
+  },
+  "activerecord": "sqlite3",
+  "sqlite3": {
+    "dbfile": "rbitter.sqlite"
+  },
+  "mysql2": {
+    "host": "localhost",
+    "port": 3306,
+    "dbname": "archive",
+    "username": "",
+    "password": ""
+  },
+  "media_downloader": {
+    "large_image": true,
+    "cacert_path": "/cacerts/cacert.pem",
+    "download_dir": "imgs/"
+  },
+  "xmlrpc": {
+    "enable": true,
+    "bind_host": "0.0.0.0",
+    "bind_port": 1400,
+    "auth": {
+      "username": "username",
+      "password": "password"
+    },
+    "handles": ["/path/to/handles"]
+  }
+}
+ENDOFJSON
+end

data/lib/rbitter/dlthread.rb

@@ -1,66 +1,66 @@
-# encoding: utf-8
-
-require "net/http"
-require "openssl"
-
-module Rbitter
-  class DLThread
-    def initialize(dlfolder, cacert_path)
-      @dest = dlfolder
-      if not File.directory?(dlfolder)
-        warn "[dlthread] Given download location is not available for downloading."
-        warn "[dlthread] Current Dir is used instead."
-        @dest = "./"
-      end
-
-      @cacert = cacert_path
-    end
-
-    def execute_urls(urls)
-      urls.each { |url|
-        download_once(url)
-      }
-    end
-
-    private
-    def download_once(url)
-      download_task = Thread.new {
-        uri = URI.parse(url)
-        http = Net::HTTP.new(uri.host, uri.port)
-        if uri.scheme.downcase == 'https'
-          http.use_ssl = true
-          http.ca_path = @cacert_path
-          # XXX: Fix this soon as possible.
-          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-        end
-
-        http.request_get(uri.path) { |res|
-          case res
-          when Net::HTTPOK
-            fname = File.basename(uri.path)
-            if fname.nil? or fname.size < 1
-              fname = uri.path.gsub(/\//, "_")
-            end
-            puts "[fetch] remote: #{url} => local: #{fname}"
-            File.open(@dest+"/"+fname, "wb") { |file|
-              res.read_body { |chunk|
-                file.write(chunk)
-              }
-            }
-          end
-        }
-      }
-
-      download_task.run
-    end
-  end
-end
-
-# XXX: Should be moved to dlthread_spec.rb
-=begin
-if __FILE__ == $0
-  t = DLThread.new(".")
-  t.execute_urls(["https://www.google.co.kr/images/nav_logo195.png"])
-  sleep 4
-end
-=end
+# encoding: utf-8
+
+require "net/http"
+require "openssl"
+
+module Rbitter
+  class DLThread
+    def initialize(dlfolder, cacert_path, large_flag)
+      @dest = dlfolder
+      if not File.directory?(dlfolder)
+        warn "[dlthread] Given download location is not available for downloading."
+        warn "[dlthread] Fallback to current directory."
+        @dest = "./"
+      end
+
+      @cacert = cacert_path
+      if large_flag.nil?
+        @large_image = false
+      else
+        @large_image = large_flag
+      end
+
+      @pool = Array.new
+    end
+
+    def <<(url_array)
+      download_task = Thread.new {
+        url_array.each { |url|
+          uri = URI.parse(@large_image ? url + ":large" : url) 
+          http = Net::HTTP.new(uri.host, uri.port)
+          if uri.scheme.downcase == 'https'
+            http.use_ssl = true
+            http.ca_path = @cacert
+          end
+
+          http.request_get(uri.path) { |res|
+            case res
+            when Net::HTTPOK
+              fname = File.basename(url)
+
+              puts "[fetch] remote: #{uri.path} => local: #{fname}"
+              open(File.join(@dest, fname), "wb") { |file|
+                res.read_body { |chunk| file.write(chunk) }
+              }
+            end
+          }
+        }
+      }
+
+      @pool.push download_task
+    end
+
+    def job_cleanup
+      until @pool.empty?
+        puts "[dlthread] Thread forceful cleaning up [remains: #{@pool.length}]"
+        
+        dlthrd = @pool.shift
+
+        if dlthrd.alive?
+          dlthrd.terminate
+          dlthrd.join
+        end
+      end
+    end
+  end
+end

data/lib/rbitter/env.rb

@@ -1,59 +1,62 @@
-# encoding: utf-8
-
-require "json"
-
-module Rbitter
-  @internal_configuration = {}
-
-  class ConfigurationFileError < StandardError; end
-
-  module_function
-  def env
-    @internal_configuration
-  end
-
-  def env_reset
-    @internal_configuration.clear
-  end
-
-  def config_load path
-    open(path, 'r') { |io|
-      @internal_configuration = JSON.parse(io.read)
-    }
-    true
-  end
-
-  def config_initialize json_path=nil
-    @internal_configuration = JSON.parse("{}")
-
-    unless json_path.nil?
-      begin
-        config_load(json_path)
-        # TODO: Configuration validation
-        return @internal_configuration
-      rescue => e
-        fail ConfigurationFileError, "Provided configuration can not be loaded. (#{json_path})"
-      end
-    end
-
-    # Configuration default location
-    # Priorities
-    # 1. (current_dir)/config.json
-    # 2. (current_dir)/.rbitter/config.json
-    locations = Array.new
-    base_locations = ["config.json", ".rbitter/config.json"]
-
-    base_locations.each { |bloc|
-        locations.push File.join(Dir.pwd, bloc)
-    }
-
-    for location in locations
-      next unless File.file?(location)
-      break if config_load(location)
-    end
-
-    if @internal_configuration.empty?
-      fail ConfigurationFileError, "Can not load any configuration in [#{locations.join(', ')}]"
-    end
-  end
-end
+# encoding: utf-8
+
+require "json"
+
+module Rbitter
+  @@env = Hash.new
+
+  class ConfigFileError < StandardError; end
+
+  def self.[](k)
+    @@env[k]
+  end
+
+  module_function
+  def env
+    @@env
+  end
+
+  def env_reset
+    @@env.clear
+  end
+
+  def env_validate?
+    # TODO: Add validator
+    true
+  end
+
+  def config_initialize json_path=nil
+    env_reset
+
+    unless json_path.nil?
+      begin
+        open(json_path, 'r') { |file|
+          @@env = JSON.parse(file.read)
+        }
+
+        return @@env if env_validate?
+        fail StandardError, "Invalid configuration"
+      rescue => e
+        fail ConfigFileError, "Load Failure (#{json_path}): #{e.to_s}"
+      end
+    end
+
+    # Configuration default location
+    # 1. (current_dir)/config.json
+    # 2. (current_dir)/.rbitter/config.json
+    locations = ["config.json", ".rbitter/config.json"]
+    locations.collect! { |base| File.join(Dir.pwd, base) }
+
+    for location in locations
+      next unless File.file?(location)
+      open(location, 'r') { |file|
+        @@env = JSON.parse(file.read)
+      }
+      break if env_validate?
+    end
+
+    if @@env.empty?
+      fail ConfigFileError, "Can not load any configuration in [#{locations.join(', ')}]"
+    end
+  end
+end

data/lib/rbitter/libtwitter_connection_override.rb

@@ -1,46 +1,46 @@
-# encoding: utf-8
-
-require 'http/parser'
-require 'openssl'
-require 'resolv'
-
-module Twitter
-  module Streaming
-    class Connection
-      MODIFIED = true
-      attr_reader :tcp_socket_class, :ssl_socket_class
-
-      def initialize(options = {})
-        @tcp_socket_class = options.fetch(:tcp_socket_class) { TCPSocket }
-        @ssl_socket_class = options.fetch(:ssl_socket_class) { OpenSSL::SSL::SSLSocket }
-      end
-
-      def stream(request, response)
-        client_context = OpenSSL::SSL::SSLContext.new
-        client = @tcp_socket_class.new(Resolv.getaddress(request.uri.host), request.uri.port)
-        ssl_client = @ssl_socket_class.new(client, client_context)
-        ssl_client.connect
-        request.stream(ssl_client)
-
-        loop {
-          begin
-            body = ssl_client.read_nonblock(1024) # rubocop:disable AssignmentInCondition, WhileUntilModifier
-            response << body
-          rescue IO::WaitReadable
-            # The reason for setting 90 seconds as a timeout is documented on:
-            # https://dev.twitter.com/streaming/overview/connecting
-            r, w, e = IO.select([ssl_client], [], [], 90)
-            if r.nil?
-              # If timeout occurs
-              ssl_client.close
-              raise Twitter::Error::ServerError.new("Connection stalled")
-            else
-              # If socket is readable
-              retry
-            end
-          end
-        }
-      end
-    end
-  end
+# encoding: utf-8
+
+require 'http/parser'
+require 'openssl'
+require 'resolv'
+
+module Twitter
+  module Streaming
+    class Connection
+      MODIFIED = true
+      attr_reader :tcp_socket_class, :ssl_socket_class
+
+      def initialize(options = {})
+        @tcp_socket_class = options.fetch(:tcp_socket_class) { TCPSocket }
+        @ssl_socket_class = options.fetch(:ssl_socket_class) { OpenSSL::SSL::SSLSocket }
+      end
+
+      def stream(request, response)
+        client_context = OpenSSL::SSL::SSLContext.new
+        client = @tcp_socket_class.new(Resolv.getaddress(request.uri.host), request.uri.port)
+        ssl_client = @ssl_socket_class.new(client, client_context)
+        ssl_client.connect
+        request.stream(ssl_client)
+
+        loop {
+          begin
+            body = ssl_client.read_nonblock(1024) # rubocop:disable AssignmentInCondition, WhileUntilModifier
+            response << body
+          rescue IO::WaitReadable
+            # The reason for setting 90 seconds as a timeout is documented on:
+            # https://dev.twitter.com/streaming/overview/connecting
+            r, w, e = IO.select([ssl_client], [], [], 90)
+            if r.nil?
+              # If timeout occurs
+              ssl_client.close
+              raise Twitter::Error::ServerError.new("Connection stalled")
+            else
+              # If socket is readable
+              retry
+            end
+          end
+        }
+      end
+    end
+  end
 end