rbitter 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3ca2dd0e8a08659416b4e1ba154375673bfe697d
-  data.tar.gz: d60daa351fef6fdfa8cc3fdb444b31e10a820c80
+  metadata.gz: c258b870ebf4e8dd9304ce2cfc588cf720fba7c1
+  data.tar.gz: 0a86c5194600ef1964ec08ae923e8abd9a6b7740
 SHA512:
-  metadata.gz: 251d4bda5d89583701ca2ecb3b52c5cafeab42b6e3f01d90692e2457d48f53e22ff9cda790daed72b63088cd077c4ba7ba27e8dd0751e9d2b5f1dbbf68dd1286
-  data.tar.gz: cb04a2013000e2581b5b8c6dd5b30b15058c6f70f995fbde2dbf31733e78a23c7b8526cafc203cae9d0125f16752c8728cc45b3103f28a83e2d52dbac153118e
+  metadata.gz: 211b5214c71b22df1d0d0f944760bc8f3c2518e1e9552d044e72c3606bb4fa158422da7330406ecd8885a0ddcf6defe416606176efa75ed16836c9f316cefb6a
+  data.tar.gz: d6f73a9ca34b941c0de31a4153a0b9144425dd0279b699f9aa60ff35c6bcbe312799f03ec6ff3c62d2021bc1ed380885aa072437cfa723e115d686c486f9dcbc

data/.gitignore

@@ -17,5 +17,6 @@ mkmf.log
 *.gem
 *.json
 *.sqlite
+*.png
 .rbitter
 .rbitter/*

data/README.md

@@ -30,6 +30,11 @@ Put your customized config.json to one of below locations.
 1. ./config.json (current folder)
 2. ./.rbitter/config.json (current folder)
 
+### XMLRPC handlers ###
+To get XMLRPC handlers, visit [Rbitter XMLRPC](https://github.com/nidev/rbitter-rpchandles).
+
+You can find the document to write your own XMLRPC handler.
+
 ## Set up and run ##
 With config.json,
 
@@ -42,9 +47,6 @@ With config.json,
 ```bash
 $ rbitter serve
 ```
-## XMLRPC API ##
-
-See XMLRPC.md
 
 ## TODO ##
 * Streaming Client

data/lib/rbitter.rb

@@ -6,6 +6,7 @@ require "rbitter/arcserver"
 require "rbitter/env"
 require "rbitter/console"
 require "rbitter/xmlrpc"
+require "rbitter/override"
 
 module Rbitter
   BOOTSTRAP_ARGS = ['configure', 'console', 'help', 'logs', 'serve']
@@ -26,29 +27,6 @@ module Rbitter
     puts "`- logs     : Show Rbitter internal logs"
   end
 
-  def self.prebootstrap
-    # Due to stalled socket problem, If unpatched twitter gem is installed.
-    # Twitter::Streaming::Connection will be monkey-patched.
-    patch_required = false
-
-    if Twitter::Version.const_defined?(:MAJOR)
-      b5_version = Twitter::Version::MAJOR * 10000
-      + Twitter::Version::MINOR * 100 + Twitter::Version::PATCH
-      if b5_version <= 51400
-        warn "[rbitter] Monkey-patching Twitter::Streaming::Connection"
-        warn "[rbitter] Please upgrade twitter gem to apply streaming read timeout"
-        patch_required = true
-      end
-    else
-      b6_version = Twitter::Version.to_a
-      if b6_version[0] <= 6 and b6_version[1] <= 0 and b6_version[2] <= 0
-        patch_required = true
-      end
-    end
-
-    require "rbitter/libtwitter_connection_override" if patch_required
-  end
-
   def self.bootstrap_configs
     require "rbitter/default/config_json"
 
@@ -61,8 +39,6 @@ module Rbitter
     return nil if args.length < 1
     
     if args[0] == "serve"
-      prebootstrap
-
       Rbitter.config_initialize
       
       archive_server = Rbitter::ArcServer.new

data/lib/rbitter/arcserver.rb

@@ -15,10 +15,16 @@ module Rbitter
     LOG_NORMAL = 0
     LOG_INIT = 1
     LOG_HALT = 2
+    LOG_ERROR = 4
 
     def initialize(xmlrpcd_class = Rbitter::RPCServer)
       @xmlrpcd_class = xmlrpcd_class
+      @dt = DLThread.new(
+        Rbitter['media_downloader']['download_dir'],
+        Rbitter['media_downloader']['large_image'])
+    end
 
+    def arsupport_init
       ARSupport.connect_database
 
       if not ARSupport.prepared?
@@ -31,10 +37,10 @@ module Rbitter
       end
 
       ARSupport.update_database_scheme
+    end
 
-      @dt = DLThread.new(
-        Rbitter['media_downloader']['download_dir'],
-        Rbitter['media_downloader']['large_image'])
+    def arsupport_halt
+      ARSupport.disconnect_database
     end
 
     def xmlrpcd_start
@@ -72,21 +78,38 @@ module Rbitter
         :fav_count => 0})
     end
 
-    def write_init_marker
+    def mark_init
       mark(LOG_INIT, "Archiving service started")
     end
 
-    def write_halt_marker
+    def mark_halt
       mark(LOG_HALT, "Archiving service halted")
     end
 
+    def mark_error(exception_string, err_msg)
+      mark(LOG_ERROR, "Errored (#{exception_string}, #{err_msg}")
+    end
+
+    def resurrect_loop?
+      if Rbitter.env['twitter']['connection']['reconnect']
+        puts "[rbitter] Try to reconnect..."
+        sleep Rbitter.env['twitter']['connection']['timeout_secs']
+        true
+      else
+        puts "[rbitter] Give up!"
+        false
+      end
+    end
+
     def main_loop(streaming_adapter = Rbitter::StreamClient)
       xmlrpcd_start if Rbitter['xmlrpc']['enable']
 
+      arsupport_init
+
       begin
-        write_init_marker
+        mark_init
 
-        streaming_adapter.new(Rbitter['twitter'].dup).run { |a|
+        streaming_adapter.new(Rbitter['twitter']).run { |a|
           @dt << a['media_urls']
 
           record = Record.find_or_initialize_by(tweetid: a['tweetid'])
@@ -106,25 +129,37 @@ module Rbitter
       rescue Interrupt => e
         puts ""
         puts "Interrupted..."
+        mark_error(e.to_s, "(exit) SIGINT - interrupted by user")
       rescue Twitter::Error::Unauthorized => e
         warn "Twitter access unauthorized:"
         warn "  Possible solutions"
         warn "  1. Configure Twitter token on config.json"
         warn "  2. Check system time (Time is important on authentication)"
         warn "  3. Check Twitter account status"
-      rescue Twitter::Error::ServerError, Resolv::ResolvError => e
-        puts "Service unavailable now. Retry in 5 second..."
-        sleep 5
-        retry
+      rescue Twitter::Error::ServerError => e
+        puts "Service unavailable now. Retry in 5 seconds..."
+        mark_error(e.to_s, "(retry) Twitter server unavailable / Timeout")
+
+        retry if resurrect_loop?
+      rescue Resolv::ResolvError, Errno::ECONNABORTED,
+        Errno::ECONNREFUSED, Errno::ECONNRESET => e
+        puts "Network problem. Retry in 5 seconds..."
+        mark_error(e.to_s, "(retry) Network problem")
+
+        retry if resurrect_loop?
       rescue Twitter::Error => e
         warn "Twitter Error: #{e.inspect}"
-        warn "Main loop of ArcServer halted."
+        warn "Rbitter halts due to Twitter::Error"
+        mark_error(e.to_s, "(exit) Twitter Error")
       ensure
         xmlrpcd_stop if Rbitter['xmlrpc']['enable']
         @dt.job_cleanup
 
-        write_halt_marker
+        mark_halt
       end
+
+      arsupport_halt
     end
+
   end
 end

data/lib/rbitter/default/config_json.rb

@@ -5,7 +5,11 @@ module Rbitter
     "consumer_key": "",
     "consumer_secret": "",
     "access_token": "",
-    "access_token_secret": ""
+    "access_token_secret": "",
+    "connection": {
+      "reconnect": true,
+      "timeout_secs": 5
+    }
   },
   "activerecord": "sqlite3",
   "sqlite3": {

data/lib/rbitter/env.rb

@@ -1,11 +1,13 @@
 # encoding: utf-8
 
 require "json"
+require "rbitter/default/config_json"
 
 module Rbitter
   @@env = Hash.new
 
   class ConfigFileError < StandardError; end
+  class MissingFieldError < StandardError; end
 
   def self.[](k)
     @@env[k]
@@ -20,9 +22,60 @@ module Rbitter
     @@env.clear
   end
 
-  def env_validate?
-    # TODO: Add validator
-    true
+  def env_listfields hash
+    path_stack = ['']
+    generated = []
+
+    until path_stack.empty?
+      path = path_stack.pop
+
+      if path == ''
+        o = hash
+      else
+        nodes = path.strip.split('->')
+        o = hash
+        until nodes.empty?
+          o = o[nodes.shift]
+        end
+      end
+
+      o.each_key { |k|
+        if o[k].is_a?(Hash)
+          path_stack << "#{k}" if path.empty?
+          path_stack << path + "->#{k}" unless path.empty?
+        else
+          generated << "#{k}" if path.empty?
+          generated << path + "->#{k}" unless path.empty?
+        end
+      }
+    end
+
+    generated
+  end
+
+  def env_valid?
+    defaults = env_listfields(JSON.parse(DEFAULT_CONFIG_JSON))
+    currents = env_listfields(@@env)
+    not_errored = true
+
+    # Cross checking (2 phases)
+    # In current exists, default does not: redundant configuration
+    # Level: warning since it is not utilized at all.
+    currents.each { |conf|
+      unless defaults.include?(conf)
+        warn "[config.json] Unused config: #{conf}. You can safely remove it."
+      end
+    }
+
+    # In default exists, current does not: missing configuration
+    # Level: error and program should stop. (return false for this)
+    defaults.each { |conf|
+      unless currents.include?(conf)
+        warn "[config.json] Config not found: #{conf}. Invalid configuration!"
+        not_errored = false
+      end
+    }
+    not_errored
   end
 
   def config_initialize json_path=nil
@@ -34,7 +87,7 @@ module Rbitter
           @@env = JSON.parse(file.read)
         }
 
-        return @@env if env_validate?
+        return @@env if env_valid?
         fail StandardError, "Invalid configuration"
       rescue => e
         fail ConfigFileError, "Load Failure (#{json_path}): #{e.to_s}"
@@ -52,11 +105,12 @@ module Rbitter
       open(location, 'r') { |file|
         @@env = JSON.parse(file.read)
       }
-      break if env_validate?
+      break unless @@env.empty?
     end
 
-    if @@env.empty?
-      fail ConfigFileError, "Can not load any configuration in [#{locations.join(', ')}]"
-    end
+    fail ConfigFileError, "No config.json on #{locations.join(' or ')}" if @@env.empty?
+    fail ConfigFileError, "Configuration outdated. Please see above messages to update it" if not env_valid?
+
+    puts "[config.json] Loaded configuration is valid. good to go!"
   end
 end

data/lib/rbitter/override.rb

@@ -0,0 +1,47 @@
+=begin
+This module gives special workarounds for some issues.
+
+Problems are bypassed by monkey-patching.
+
+As soon as a problem is resolved, patch should be removed.
+
+**** Known issues ****
+
+1. gem/twitter
+Location : override/gem/twitter/connection.rb
+Maintain : until below issue is fixed
+Reference: https://github.com/sferik/twitter/pull/669
+
+Gem 'twitter' does not handle streaming timeout. By applying this,
+reading timeout works and Rbitter can handle timeouts.
+
+2. gem/rubysl-socket
+Location : override/gem/rubysl-socket/socket.rb
+Maintain : until ip_address_list is implemented
+Reference: https://github.com/rubysl/rubysl-socket/pull/9
+
+With ipv6 environment, Resolv#use_ipv6? (in rubysl-resolv gem) checks 
+Socket.ip_address_list. This is not implemented at all with rubysl-socket-2.0.1.
+NoMethodError exception is raised instead of NotImplementedError.
+
+By applying this, Socket.ip_address_list is implemented and the method throws
+NotImplementedError exception.
+
+=end
+
+def gem_twitter_patcher
+  require 'rbitter/override/gems/twitter/connection'
+end
+
+if Twitter::Version.const_defined?(:MAJOR)
+  b5_version = Twitter::Version::MAJOR * 10000
+  + Twitter::Version::MINOR * 100 + Twitter::Version::PATCH
+  gem_twitter_patcher if b5_version <= 51400
+else
+  b6_version = Twitter::Version.to_a
+  if b6_version[0] <= 6 and b6_version[1] <= 0 and b6_version[2] <= 0
+    gem_twitter_patcher
+  end
+end
+
+require 'rbitter/override/gems/rubysl-socket/socket' if RUBY_ENGINE == 'rbx'

data/lib/rbitter/override/gems/rubysl-socket/socket.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+require 'socket'
+
+class Socket
+  def self.ip_address_list
+    fail NotImplementedError
+  end
+end

data/lib/rbitter/records.rb

@@ -63,6 +63,12 @@ module ARSupport
     end
   end
 
+  def disconnect_database
+    if ActiveRecord::Base.connected?
+      ActiveRecord::Base.connection.close
+    end
+  end
+
   def update_database_scheme
     current_version = ActiveRecord::Migrator.current_version
     if current_version < SCHEME_VERSION

data/lib/rbitter/version.rb

@@ -1,6 +1,6 @@
 module Rbitter
   PRODUCT_NAME = "Rbitter"
-  VERSION = "0.1.2"
+  VERSION = "0.2.0"
 
   def major
     VERSION.match(/^([0-9]+)\./)[1]

data/spec/rbitter/env_spec.rb

@@ -53,10 +53,24 @@ describe Rbitter do
     end
   end
 
-  context 'when env_validator validates loaded configuration' do
-    # TODO: Perform test with spec/config/default.json
-    # TODO: Adding configuration validator on env.rb
-  end
+  context "when 'env_valid?' validates loaded configuration" do
+    before(:all) do
+      Rbitter.bootstrap(['configure'])
+      expect(File.file?('config.json')).to be(true)
+    end
+
+    it 'gets \'false\' for empty configuration' do
+      Rbitter.env_reset
+      expect(Rbitter.env_valid?).to be(false)
+    end
 
-  # TODO: Perform test with spec/config/default.json
+    it "gets \'true\' with default and another default" do
+      Rbitter.config_initialize
+      expect(Rbitter.env_valid?).to be(true)
+    end
+
+    after(:all) do
+      File.delete('config.json')
+    end
+  end
 end

data/spec/rbitter/{libtwitter_connection_override_spec.rb → override/gems/twitter/connection_spec.rb}

@@ -1,5 +1,5 @@
 # encoding: utf-8
-require "rbitter/libtwitter_connection_override"
+require "rbitter/override/gems/twitter/connection"
 
 describe Rbitter do
   it 'overrides twitter gem' do

data/spec/rbitter_spec.rb

@@ -22,10 +22,6 @@ describe Rbitter do
       Rbitter.bootstrap(['configure'])
     end
 
-    it 'executes prebootstrap properly' do
-      Rbitter.prebootstrap
-    end
-
     it 'bootstraps nothing when empty ARGV is given'  do
       Rbitter.bootstrap([])
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbitter
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Nidev Plontra
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-17 00:00:00.000000000 Z
+date: 2015-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: twitter
@@ -152,7 +152,6 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- XMLRPC.md
 - bin/rbitter
 - lib/rbitter.rb
 - lib/rbitter/arcserver.rb
@@ -160,7 +159,9 @@ files:
 - lib/rbitter/default/config_json.rb
 - lib/rbitter/dlthread.rb
 - lib/rbitter/env.rb
-- lib/rbitter/libtwitter_connection_override.rb
+- lib/rbitter/override.rb
+- lib/rbitter/override/gems/rubysl-socket/socket.rb
+- lib/rbitter/override/gems/twitter/connection.rb
 - lib/rbitter/records.rb
 - lib/rbitter/records_migrate/.keep
 - lib/rbitter/records_migrate/20150327_add_index.rb
@@ -180,7 +181,8 @@ files:
 - spec/rbitter/default/config_json_spec.rb
 - spec/rbitter/dlthread_spec.rb
 - spec/rbitter/env_spec.rb
-- spec/rbitter/libtwitter_connection_override_spec.rb
+- spec/rbitter/override/gems/rubysl-socket/socket_spec.rb
+- spec/rbitter/override/gems/twitter/connection_spec.rb
 - spec/rbitter/records_spec.rb
 - spec/rbitter/streaming_spec.rb
 - spec/rbitter/version_spec.rb
@@ -224,7 +226,8 @@ test_files:
 - spec/rbitter/default/config_json_spec.rb
 - spec/rbitter/dlthread_spec.rb
 - spec/rbitter/env_spec.rb
-- spec/rbitter/libtwitter_connection_override_spec.rb
+- spec/rbitter/override/gems/rubysl-socket/socket_spec.rb
+- spec/rbitter/override/gems/twitter/connection_spec.rb
 - spec/rbitter/records_spec.rb
 - spec/rbitter/streaming_spec.rb
 - spec/rbitter/version_spec.rb

data/XMLRPC.md

@@ -1,19 +0,0 @@
-# What is RPC handle ? #
-## Commands ##
-### Authentication ###
-### Revoke Authentication Token ###
-### Echo ###
-### Last Active ###
-### Retriever ###
-### Statistic ###
-# How to write own RPC handle? #
-RPC handle is a Ruby class. Writing a method in Ruby class, that's it. Names of methods are treated as XMLRPC command.
-
-When you write a new class for your own RPC handle, you must inherit either Auth or NoAuth class from rpc/base.rb.
-
-* class Auth < Object: Methods in a Ruby class inheriting Auth requires *auth_key* to access.
-* class NoAuth < Object: Methods in a Ruby class inheriting NoAuth doesn't require *auth_key* and these XMLRPC commands can be called by anonymous user.
-
-Filename should start with 'rh_'. It's prefix to be autoloaded by xmlrpc.rb.
-
-Refer rpc/rh_echo.rb as an example.