rbitter 0.2.1 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9f7b80e94710352473dc6934be1af715dd8be3ff
-  data.tar.gz: 6a9cf2305b5d255311a8b6d83f660b1e3fd38a8e
+  metadata.gz: 7e12755836fb33a945c9f9e6c48dc911fc7ac1a5
+  data.tar.gz: 15d8892a885d452f71fa90c58a018693e7c582f9
 SHA512:
-  metadata.gz: d78351f985d2aba67be9fcb1c2b5b4cdb168bee90b7b294b4606914731029dd3f48b253978f8d3b3760bab96025428e7a3d8f8b414b3a4b6379f544a90153460
-  data.tar.gz: a1c84643f11a6964e070ffe52964aafb8c577ed550601434a473332441dc16596565735ca784eff921bc94adfa66bf07aaba9cc7a027cb2d566d07e00e2ea3b9
+  metadata.gz: d8ae6dc9b1dfab693a6dedc5bb1eefb6d6e0c1c8ec4455abfbd2cf4f0fc8c365ffa172e76f59496aef7cd6e802f5945d308532460512251662568055d08b0265
+  data.tar.gz: 88992b4711bffd9e80aafbd01b3d98a7ff13c0ddec852380544f939b636a64ecf5c7929284066a26814c9471b0fd51d5eb9b30d5d032f8f4ae573d17538e2c51

data/.gitignore

@@ -20,3 +20,4 @@ mkmf.log
 *.png
 .rbitter
 .rbitter/*
+.ruby-version

data/Gemfile

@@ -1,7 +1,6 @@
 source "https://rubygems.org"
 
 group :test do
-  gem 'rspec', '~>3.0'
   gem 'simplecov', :require => false, :group => :test
   gem 'coveralls', :require => false
 end

data/lib/rbitter/arcserver.rb

@@ -114,6 +114,7 @@ module Rbitter
 
         streaming_adapter.new(Rbitter['twitter']).run { |a|
           @dt << a['media_urls']
+          draw "[rbitter] saving @#{a['screen_name']}:#{a['tweetid']}"
 
           record = Record.find_or_initialize_by(tweetid: a['tweetid'])
           record.update({:marker => 0,
@@ -128,7 +129,7 @@ module Rbitter
             :fav_count => a['fav_count']})
 
           record.save
-          draw "[rbitter] saving tweet: #{a['tweetid']}"
+          draw "[rbitter] saved @#{a['screen_name']}:#{a['tweetid']}"
         }
       rescue Interrupt => e
         puts ""
@@ -146,7 +147,7 @@ module Rbitter
 
         retry if resurrect_loop?
       rescue Resolv::ResolvError, Errno::ECONNABORTED,
-        Errno::ECONNREFUSED, Errno::ECONNRESET => e
+        Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ETIMEDOUT  => e
         puts "Network problem. Retry in 5 seconds..."
         mark_error(e.to_s, "(retry) Network problem")
 

data/lib/rbitter/dlthread.rb

@@ -5,6 +5,8 @@ require "openssl"
 
 module Rbitter
   class DLThread
+    MAX_THREADS = 20
+
     def initialize(dlfolder, large_flag)
       @dest = dlfolder
       if not File.directory?(dlfolder)
@@ -23,27 +25,34 @@ module Rbitter
     end
 
     def <<(url_array)
-      download_task = Thread.new {
+      if @pool.length >= MAX_THREADS
+        job_cleanup
+      end
+
+      download_task = Thread.new do
         url_array.each { |url|
           uri = URI.parse(@large_image ? url + ":large" : url) 
-          ssl = uri.scheme.downcase == 'https'
-
-          Net::HTTP.start(uri.host, uri.port, :use_ssl => ssl) { |h|
-            req = Net::HTTP::Get.new uri.request_uri
-            h.request(req) { |res|
-              case res
-              when Net::HTTPOK
-                fname = File.basename(url)
-
-                puts "[fetch] remote: #{uri.path} => local: #{fname}"
-                open(File.join(@dest, fname), "wb") { |file|
-                  res.read_body { |chunk| file.write(chunk) }
-                }
-              end
+
+          begin
+            Net::HTTP.start(uri.host, uri.port, :use_ssl => uri.scheme.downcase == 'https') { |h|
+              req = Net::HTTP::Get.new uri.request_uri
+              h.request(req) { |res|
+                case res
+                when Net::HTTPOK
+                  fname = File.basename(url)
+
+                  puts "[fetch] remote: #{uri.path} => local: #{fname}"
+                  open(File.join(@dest, fname), "wb") { |file|
+                    res.read_body { |chunk| file.write(chunk) }
+                  }
+                end
+              }
             }
-          }
+          rescue => e
+            puts "[dlthread] exception from thread -> #{e.to_s}"
+          end
         }
-      }
+      end
 
       @pool.push download_task
     end
@@ -52,12 +61,10 @@ module Rbitter
       until @pool.empty?
         dlthrd = @pool.shift
 
-        if dlthrd.alive?
-          puts "[dlthread] Thread forceful cleaning up [remains: #{@pool.length}]"
-          dlthrd.terminate
-          dlthrd.join
+        if dlthrd.join(5).nil?
+          puts "[dlthread] #{dlthrd.to_s} is still running. (timeout: 5sec)"
         end
       end
     end
   end
-end
+end

data/lib/rbitter/env.rb

@@ -63,7 +63,7 @@ module Rbitter
     # Level: warning since it is not utilized at all.
     currents.each { |conf|
       unless defaults.include?(conf)
-        warn "[config.json] Unused config: #{conf}. You can safely remove it."
+        warn "[config.json] Unsupported field: #{conf}"
       end
     }
 
@@ -71,7 +71,7 @@ module Rbitter
     # Level: error and program should stop. (return false for this)
     defaults.each { |conf|
       unless currents.include?(conf)
-        warn "[config.json] Config not found: #{conf}. Invalid configuration!"
+        warn "[config.json] Error! Missing field: #{conf}"
         not_errored = false
       end
     }
@@ -88,9 +88,9 @@ module Rbitter
         }
 
         return @@env if env_valid?
-        fail StandardError, "Invalid configuration"
+        fail StandardError, "Void configuration. Maybe failure on load?"
       rescue => e
-        fail ConfigFileError, "Load Failure (#{json_path}): #{e.to_s}"
+        fail ConfigFileError, "Config error on (#{json_path}): #{e.to_s}"
       end
     end
 
@@ -111,6 +111,6 @@ module Rbitter
     fail ConfigFileError, "No config.json on #{locations.join(' or ')}" if @@env.empty?
     fail ConfigFileError, "Configuration outdated. Please see above messages to update it" if not env_valid?
 
-    puts "[config.json] Loaded configuration is valid. good to go!"
+    puts "[config.json] JSON data structure is safe to go."
   end
 end

data/lib/rbitter/progress.rb

@@ -10,7 +10,10 @@ module Rbitter
     def draw sentence
       $> << sentence
       if sentence.length < @@last_draw
-        $> << " "*(@@last_draw - sentence.length)
+	clear_char_len = @@last_draw - sentence.length
+	clear_char_len.times {
+	  $> << " "
+	}
       end
       @@last_draw = sentence.length
       putback

data/lib/rbitter/records.rb

@@ -47,7 +47,10 @@ module ARSupport
           timeout: 10000) # Long timeout for slow computer
       end
     elsif Rbitter['activerecord'] == 'mysql2'
-      Jdbc::MySQL.load_driver if RUBY_PLATFORM == 'java'
+      if RUBY_PLATFORM == 'java'
+        require 'jdbc/mysql'
+        Jdbc::MySQL.load_driver
+      end
       
       ActiveRecord::Base.establish_connection(
         adapter: (RUBY_PLATFORM == 'java' ? 'jdbcmysql' : 'mysql2'),

data/lib/rbitter/version.rb

@@ -1,6 +1,6 @@
 module Rbitter
   PRODUCT_NAME = "Rbitter"
-  VERSION = "0.2.1"
+  VERSION = "0.2.3"
 
   def major
     VERSION.match(/^([0-9]+)\./)[1]

data/rbitter.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'twitter', '~> 5.14'
   spec.add_dependency 'json', '~> 1.7'
   spec.add_dependency 'ripl', '~> 0.7'
-  spec.add_dependency 'activerecord', '~> 4.0'
+  spec.add_dependency 'activerecord', '~> 4.2'
 
   if RUBY_PLATFORM == 'java'
     spec.platform = 'java'
@@ -38,9 +38,9 @@ Gem::Specification.new do |spec|
 
     spec.add_dependency 'sqlite3', '~> 1.3'
     spec.add_dependency 'mysql2', '~> 0.3'
-    spec.add_dependency 'activerecord-mysql2-adapter', '~> 0.0.3'
   end
 
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.3"
 end

metadata

@@ -1,24 +1,24 @@
 --- !ruby/object:Gem::Specification
 name: rbitter
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.3
 platform: ruby
 authors:
 - Nidev Plontra
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-07 00:00:00.000000000 Z
+date: 2015-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: twitter
-  prerelease: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.14'
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -26,13 +26,13 @@ dependencies:
         version: '5.14'
 - !ruby/object:Gem::Dependency
   name: json
-  prerelease: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.7'
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -40,13 +40,13 @@ dependencies:
         version: '1.7'
 - !ruby/object:Gem::Dependency
   name: ripl
-  prerelease: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.7'
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -54,27 +54,27 @@ dependencies:
         version: '0.7'
 - !ruby/object:Gem::Dependency
   name: activerecord
-  prerelease: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '4.2'
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '4.2'
 - !ruby/object:Gem::Dependency
   name: sqlite3
-  prerelease: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -82,60 +82,60 @@ dependencies:
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: mysql2
-  prerelease: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.3'
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.3'
 - !ruby/object:Gem::Dependency
-  name: activerecord-mysql2-adapter
-  prerelease: false
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.3
-  type: :runtime
+        version: '1.6'
+  type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.3
+        version: '1.6'
 - !ruby/object:Gem::Dependency
-  name: bundler
-  prerelease: false
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '10.0'
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '10.0'
 - !ruby/object:Gem::Dependency
-  name: rake
-  prerelease: false
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '3.3'
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '3.3'
 description: Rbitter archives all tweets appeared on user streaming using ActiveRecord.
   XMLRPC is used to serve archived tweets and useful features
 email:
@@ -216,7 +216,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Rbitter is a Twitter client specialized in archiving