rbitter 0.2.0-java → 0.2.1-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +23 -22
- data/.rspec +2 -2
- data/.travis.yml +15 -15
- data/Gemfile +12 -12
- data/LICENSE.txt +22 -22
- data/Rakefile +8 -8
- data/bin/rbitter +20 -20
- data/lib/rbitter/arcserver.rb +169 -165
- data/lib/rbitter/console.rb +93 -93
- data/lib/rbitter/default/config_json.rb +41 -41
- data/lib/rbitter/dlthread.rb +62 -62
- data/lib/rbitter/env.rb +116 -116
- data/lib/rbitter/override/gems/rubysl-socket/socket.rb +8 -8
- data/lib/rbitter/override/gems/twitter/connection.rb +45 -45
- data/lib/rbitter/override.rb +47 -47
- data/lib/rbitter/progress.rb +23 -0
- data/lib/rbitter/records.rb +127 -127
- data/lib/rbitter/records_migrate/20150327_add_index.rb +11 -11
- data/lib/rbitter/records_migrate/20150504_add_replyto_column.rb +11 -11
- data/lib/rbitter/streaming.rb +104 -104
- data/lib/rbitter/version.rb +20 -20
- data/lib/rbitter/xmlrpc.rb +3 -3
- data/lib/rbitter/xmlrpcd/base.rb +24 -24
- data/lib/rbitter/xmlrpcd/rpchandles.rb +11 -11
- data/lib/rbitter/xmlrpcd/xmlrpc_auth_server.rb +82 -82
- data/lib/rbitter/xmlrpcd/xmlrpcd.rb +69 -69
- data/lib/rbitter.rb +62 -62
- data/rbitter.gemspec +46 -46
- data/spec/config/default.json +32 -32
- data/spec/rbitter/arcserver_spec.rb +30 -30
- data/spec/rbitter/console_spec.rb +9 -9
- data/spec/rbitter/default/config_json_spec.rb +3 -3
- data/spec/rbitter/dlthread_spec.rb +8 -8
- data/spec/rbitter/env_spec.rb +76 -76
- data/spec/rbitter/override/gems/twitter/connection_spec.rb +8 -8
- data/spec/rbitter/progress_spec.rb +1 -0
- data/spec/rbitter/records_spec.rb +13 -13
- data/spec/rbitter/streaming_spec.rb +9 -9
- data/spec/rbitter/version_spec.rb +8 -8
- data/spec/rbitter/xmlrpc_spec.rb +8 -8
- data/spec/rbitter/xmlrpcd/base_spec.rb +29 -29
- data/spec/rbitter/xmlrpcd/rpchandles_spec.rb +10 -10
- data/spec/rbitter/xmlrpcd/xmlrpc_auth_server_spec.rb +8 -8
- data/spec/rbitter/xmlrpcd/xmlrpcd_spec.rb +9 -9
- data/spec/rbitter_spec.rb +38 -38
- data/spec/spec_helper.rb +39 -39
- metadata +6 -3
    
        data/lib/rbitter/console.rb
    CHANGED
    
    | @@ -1,93 +1,93 @@ | |
| 1 | 
            -
            # encoding: utf-8
         | 
| 2 | 
            -
            #
         | 
| 3 | 
            -
            # Rbitter Archive Access console (irb)
         | 
| 4 | 
            -
             | 
| 5 | 
            -
            require "xmlrpc/client"
         | 
| 6 | 
            -
            require "rbitter/version"
         | 
| 7 | 
            -
            require "ripl"
         | 
| 8 | 
            -
             | 
| 9 | 
            -
            module Rbitter
         | 
| 10 | 
            -
              class Console
         | 
| 11 | 
            -
                def initialize
         | 
| 12 | 
            -
                  puts "Rbitter console #{Rbitter::VERSION}"
         | 
| 13 | 
            -
                  help
         | 
| 14 | 
            -
                end
         | 
| 15 | 
            -
             | 
| 16 | 
            -
                def help
         | 
| 17 | 
            -
                  puts "Predefined methods:"
         | 
| 18 | 
            -
                  puts "ar - shortcut to call Rbitter::Record"
         | 
| 19 | 
            -
                  puts "connect_ar - Prepare Rbitter::Record to be ready"
         | 
| 20 | 
            -
                  puts "csv_backup - export Rbitter::Record into comma-separated values"
         | 
| 21 | 
            -
                  puts "help - to show this message again"
         | 
| 22 | 
            -
                  puts "xmlrpc - send xmlrpc command to destination"
         | 
| 23 | 
            -
                  puts "xmlrpc_dest - set destination for xmlrpc command"
         | 
| 24 | 
            -
                  puts "^D, 'exit' to exit from here"
         | 
| 25 | 
            -
                end
         | 
| 26 | 
            -
             | 
| 27 | 
            -
                def connect_ar
         | 
| 28 | 
            -
                  ARSupport.connect_database
         | 
| 29 | 
            -
                  puts "Rbitter::Record is ready."
         | 
| 30 | 
            -
                end
         | 
| 31 | 
            -
             | 
| 32 | 
            -
                def csv_backup *args
         | 
| 33 | 
            -
                  if args.length < 0
         | 
| 34 | 
            -
                    puts "Usage: csv_backup('filename.csv')"
         | 
| 35 | 
            -
                    puts "Estimated running time depends on system environment"
         | 
| 36 | 
            -
                  else
         | 
| 37 | 
            -
                    ARSupport.export_to_csv(args[0])
         | 
| 38 | 
            -
                  end
         | 
| 39 | 
            -
                end
         | 
| 40 | 
            -
             | 
| 41 | 
            -
                def ar
         | 
| 42 | 
            -
                  Rbitter::Record
         | 
| 43 | 
            -
                end
         | 
| 44 | 
            -
             | 
| 45 | 
            -
                def exit
         | 
| 46 | 
            -
                  Kernel.exit(0)
         | 
| 47 | 
            -
                end
         | 
| 48 | 
            -
                
         | 
| 49 | 
            -
                def xmlrpc_dest args={}
         | 
| 50 | 
            -
                  if args.empty?
         | 
| 51 | 
            -
                    puts "Usage: xmlrpc_dest({ :rpchost => '', :rpcpath => '', :rpcport => 1400,"
         | 
| 52 | 
            -
                    puts "                   :xmlrpc_auth_id => '', xmlrpc_auth_password => '' })"
         | 
| 53 | 
            -
                  end
         | 
| 54 | 
            -
             | 
| 55 | 
            -
                  @rpchost = args.fetch(:rpchost) { "127.0.0.1" }
         | 
| 56 | 
            -
                  @rpcpath = args.fetch(:rpcpath) { "/" }
         | 
| 57 | 
            -
                  @rpcport = args.fetch(:rpcport) { 1400 }
         | 
| 58 | 
            -
             | 
| 59 | 
            -
                  cl = XMLRPC::Client.new(@rpchost, @rpcpath, @rpcport)
         | 
| 60 | 
            -
                  @xmlrpc_cookie = "auth_key=" + cl.call('rbitter.auth',
         | 
| 61 | 
            -
                    args.fetch(:xmlrpc_auth_id) { Rbitter.env['xmlrpc']['auth'][0] },
         | 
| 62 | 
            -
                    args.fetch(:xmlrpc_auth_password) { Rbitter.env['xmlrpc']['auth'][1] } )
         | 
| 63 | 
            -
             | 
| 64 | 
            -
                  if @xmlrpc_cookie != "auth_key="
         | 
| 65 | 
            -
                    puts "Authentication completed"
         | 
| 66 | 
            -
                  else
         | 
| 67 | 
            -
                    puts "Authentication failed"
         | 
| 68 | 
            -
                  end
         | 
| 69 | 
            -
                end
         | 
| 70 | 
            -
             | 
| 71 | 
            -
                def xmlrpc *args
         | 
| 72 | 
            -
                  if args.empty?
         | 
| 73 | 
            -
                    puts "Usage: xmlrpc(command, [params in Array])"
         | 
| 74 | 
            -
                    puts "Ex) xmlrpc(\'rbitter.echo\',' [\"Hello World!\"])"
         | 
| 75 | 
            -
                    puts "Please configure XMLRPC destination with xmlrpc_dest method"
         | 
| 76 | 
            -
                    return false
         | 
| 77 | 
            -
                  end
         | 
| 78 | 
            -
             | 
| 79 | 
            -
                  cl = XMLRPC::Client.new(@rpchost, @rpcpath, @rpcport, @xmlrpc_cookie)
         | 
| 80 | 
            -
                  
         | 
| 81 | 
            -
                  if args.length <= 1 or args[1].nil?
         | 
| 82 | 
            -
                    cl.call(args[0])
         | 
| 83 | 
            -
                  else
         | 
| 84 | 
            -
                    cl.call(args[0], *args[1])
         | 
| 85 | 
            -
                  end
         | 
| 86 | 
            -
                end
         | 
| 87 | 
            -
             | 
| 88 | 
            -
                def start
         | 
| 89 | 
            -
                  Ripl.start :binding => binding
         | 
| 90 | 
            -
                end
         | 
| 91 | 
            -
              end
         | 
| 92 | 
            -
            end
         | 
| 93 | 
            -
             | 
| 1 | 
            +
            # encoding: utf-8
         | 
| 2 | 
            +
            #
         | 
| 3 | 
            +
            # Rbitter Archive Access console (irb)
         | 
| 4 | 
            +
             | 
| 5 | 
            +
            require "xmlrpc/client"
         | 
| 6 | 
            +
            require "rbitter/version"
         | 
| 7 | 
            +
            require "ripl"
         | 
| 8 | 
            +
             | 
| 9 | 
            +
            module Rbitter
         | 
| 10 | 
            +
              class Console
         | 
| 11 | 
            +
                def initialize
         | 
| 12 | 
            +
                  puts "Rbitter console #{Rbitter::VERSION}"
         | 
| 13 | 
            +
                  help
         | 
| 14 | 
            +
                end
         | 
| 15 | 
            +
             | 
| 16 | 
            +
                def help
         | 
| 17 | 
            +
                  puts "Predefined methods:"
         | 
| 18 | 
            +
                  puts "ar - shortcut to call Rbitter::Record"
         | 
| 19 | 
            +
                  puts "connect_ar - Prepare Rbitter::Record to be ready"
         | 
| 20 | 
            +
                  puts "csv_backup - export Rbitter::Record into comma-separated values"
         | 
| 21 | 
            +
                  puts "help - to show this message again"
         | 
| 22 | 
            +
                  puts "xmlrpc - send xmlrpc command to destination"
         | 
| 23 | 
            +
                  puts "xmlrpc_dest - set destination for xmlrpc command"
         | 
| 24 | 
            +
                  puts "^D, 'exit' to exit from here"
         | 
| 25 | 
            +
                end
         | 
| 26 | 
            +
             | 
| 27 | 
            +
                def connect_ar
         | 
| 28 | 
            +
                  ARSupport.connect_database
         | 
| 29 | 
            +
                  puts "Rbitter::Record is ready."
         | 
| 30 | 
            +
                end
         | 
| 31 | 
            +
             | 
| 32 | 
            +
                def csv_backup *args
         | 
| 33 | 
            +
                  if args.length < 0
         | 
| 34 | 
            +
                    puts "Usage: csv_backup('filename.csv')"
         | 
| 35 | 
            +
                    puts "Estimated running time depends on system environment"
         | 
| 36 | 
            +
                  else
         | 
| 37 | 
            +
                    ARSupport.export_to_csv(args[0])
         | 
| 38 | 
            +
                  end
         | 
| 39 | 
            +
                end
         | 
| 40 | 
            +
             | 
| 41 | 
            +
                def ar
         | 
| 42 | 
            +
                  Rbitter::Record
         | 
| 43 | 
            +
                end
         | 
| 44 | 
            +
             | 
| 45 | 
            +
                def exit
         | 
| 46 | 
            +
                  Kernel.exit(0)
         | 
| 47 | 
            +
                end
         | 
| 48 | 
            +
                
         | 
| 49 | 
            +
                def xmlrpc_dest args={}
         | 
| 50 | 
            +
                  if args.empty?
         | 
| 51 | 
            +
                    puts "Usage: xmlrpc_dest({ :rpchost => '', :rpcpath => '', :rpcport => 1400,"
         | 
| 52 | 
            +
                    puts "                   :xmlrpc_auth_id => '', xmlrpc_auth_password => '' })"
         | 
| 53 | 
            +
                  end
         | 
| 54 | 
            +
             | 
| 55 | 
            +
                  @rpchost = args.fetch(:rpchost) { "127.0.0.1" }
         | 
| 56 | 
            +
                  @rpcpath = args.fetch(:rpcpath) { "/" }
         | 
| 57 | 
            +
                  @rpcport = args.fetch(:rpcport) { 1400 }
         | 
| 58 | 
            +
             | 
| 59 | 
            +
                  cl = XMLRPC::Client.new(@rpchost, @rpcpath, @rpcport)
         | 
| 60 | 
            +
                  @xmlrpc_cookie = "auth_key=" + cl.call('rbitter.auth',
         | 
| 61 | 
            +
                    args.fetch(:xmlrpc_auth_id) { Rbitter.env['xmlrpc']['auth'][0] },
         | 
| 62 | 
            +
                    args.fetch(:xmlrpc_auth_password) { Rbitter.env['xmlrpc']['auth'][1] } )
         | 
| 63 | 
            +
             | 
| 64 | 
            +
                  if @xmlrpc_cookie != "auth_key="
         | 
| 65 | 
            +
                    puts "Authentication completed"
         | 
| 66 | 
            +
                  else
         | 
| 67 | 
            +
                    puts "Authentication failed"
         | 
| 68 | 
            +
                  end
         | 
| 69 | 
            +
                end
         | 
| 70 | 
            +
             | 
| 71 | 
            +
                def xmlrpc *args
         | 
| 72 | 
            +
                  if args.empty?
         | 
| 73 | 
            +
                    puts "Usage: xmlrpc(command, [params in Array])"
         | 
| 74 | 
            +
                    puts "Ex) xmlrpc(\'rbitter.echo\',' [\"Hello World!\"])"
         | 
| 75 | 
            +
                    puts "Please configure XMLRPC destination with xmlrpc_dest method"
         | 
| 76 | 
            +
                    return false
         | 
| 77 | 
            +
                  end
         | 
| 78 | 
            +
             | 
| 79 | 
            +
                  cl = XMLRPC::Client.new(@rpchost, @rpcpath, @rpcport, @xmlrpc_cookie)
         | 
| 80 | 
            +
                  
         | 
| 81 | 
            +
                  if args.length <= 1 or args[1].nil?
         | 
| 82 | 
            +
                    cl.call(args[0])
         | 
| 83 | 
            +
                  else
         | 
| 84 | 
            +
                    cl.call(args[0], *args[1])
         | 
| 85 | 
            +
                  end
         | 
| 86 | 
            +
                end
         | 
| 87 | 
            +
             | 
| 88 | 
            +
                def start
         | 
| 89 | 
            +
                  Ripl.start :binding => binding
         | 
| 90 | 
            +
                end
         | 
| 91 | 
            +
              end
         | 
| 92 | 
            +
            end
         | 
| 93 | 
            +
             | 
| @@ -1,41 +1,41 @@ | |
| 1 | 
            -
            module Rbitter
         | 
| 2 | 
            -
              DEFAULT_CONFIG_JSON = <<-ENDOFJSON
         | 
| 3 | 
            -
            {
         | 
| 4 | 
            -
              "twitter": {
         | 
| 5 | 
            -
                "consumer_key": "",
         | 
| 6 | 
            -
                "consumer_secret": "",
         | 
| 7 | 
            -
                "access_token": "",
         | 
| 8 | 
            -
                "access_token_secret": "",
         | 
| 9 | 
            -
                "connection": {
         | 
| 10 | 
            -
                  "reconnect": true,
         | 
| 11 | 
            -
                  "timeout_secs": 5
         | 
| 12 | 
            -
                }
         | 
| 13 | 
            -
              },
         | 
| 14 | 
            -
              "activerecord": "sqlite3",
         | 
| 15 | 
            -
              "sqlite3": {
         | 
| 16 | 
            -
                "dbfile": "rbitter.sqlite"
         | 
| 17 | 
            -
              },
         | 
| 18 | 
            -
              "mysql2": {
         | 
| 19 | 
            -
                "host": "localhost",
         | 
| 20 | 
            -
                "port": 3306,
         | 
| 21 | 
            -
                "dbname": "archive",
         | 
| 22 | 
            -
                "username": "",
         | 
| 23 | 
            -
                "password": ""
         | 
| 24 | 
            -
              },
         | 
| 25 | 
            -
              "media_downloader": {
         | 
| 26 | 
            -
                "large_image": true,
         | 
| 27 | 
            -
                "download_dir": "imgs/"
         | 
| 28 | 
            -
              },
         | 
| 29 | 
            -
              "xmlrpc": {
         | 
| 30 | 
            -
                "enable": true,
         | 
| 31 | 
            -
                "bind_host": "0.0.0.0",
         | 
| 32 | 
            -
                "bind_port": 1400,
         | 
| 33 | 
            -
                "auth": {
         | 
| 34 | 
            -
                  "username": "username",
         | 
| 35 | 
            -
                  "password": "password"
         | 
| 36 | 
            -
                },
         | 
| 37 | 
            -
                "handles": ["/path/to/handles"]
         | 
| 38 | 
            -
              }
         | 
| 39 | 
            -
            }
         | 
| 40 | 
            -
            ENDOFJSON
         | 
| 41 | 
            -
            end
         | 
| 1 | 
            +
            module Rbitter
         | 
| 2 | 
            +
              DEFAULT_CONFIG_JSON = <<-ENDOFJSON
         | 
| 3 | 
            +
            {
         | 
| 4 | 
            +
              "twitter": {
         | 
| 5 | 
            +
                "consumer_key": "",
         | 
| 6 | 
            +
                "consumer_secret": "",
         | 
| 7 | 
            +
                "access_token": "",
         | 
| 8 | 
            +
                "access_token_secret": "",
         | 
| 9 | 
            +
                "connection": {
         | 
| 10 | 
            +
                  "reconnect": true,
         | 
| 11 | 
            +
                  "timeout_secs": 5
         | 
| 12 | 
            +
                }
         | 
| 13 | 
            +
              },
         | 
| 14 | 
            +
              "activerecord": "sqlite3",
         | 
| 15 | 
            +
              "sqlite3": {
         | 
| 16 | 
            +
                "dbfile": "rbitter.sqlite"
         | 
| 17 | 
            +
              },
         | 
| 18 | 
            +
              "mysql2": {
         | 
| 19 | 
            +
                "host": "localhost",
         | 
| 20 | 
            +
                "port": 3306,
         | 
| 21 | 
            +
                "dbname": "archive",
         | 
| 22 | 
            +
                "username": "",
         | 
| 23 | 
            +
                "password": ""
         | 
| 24 | 
            +
              },
         | 
| 25 | 
            +
              "media_downloader": {
         | 
| 26 | 
            +
                "large_image": true,
         | 
| 27 | 
            +
                "download_dir": "imgs/"
         | 
| 28 | 
            +
              },
         | 
| 29 | 
            +
              "xmlrpc": {
         | 
| 30 | 
            +
                "enable": true,
         | 
| 31 | 
            +
                "bind_host": "0.0.0.0",
         | 
| 32 | 
            +
                "bind_port": 1400,
         | 
| 33 | 
            +
                "auth": {
         | 
| 34 | 
            +
                  "username": "username",
         | 
| 35 | 
            +
                  "password": "password"
         | 
| 36 | 
            +
                },
         | 
| 37 | 
            +
                "handles": ["/path/to/handles"]
         | 
| 38 | 
            +
              }
         | 
| 39 | 
            +
            }
         | 
| 40 | 
            +
            ENDOFJSON
         | 
| 41 | 
            +
            end
         | 
    
        data/lib/rbitter/dlthread.rb
    CHANGED
    
    | @@ -1,63 +1,63 @@ | |
| 1 | 
            -
            # encoding: utf-8
         | 
| 2 | 
            -
             | 
| 3 | 
            -
            require "net/http"
         | 
| 4 | 
            -
            require "openssl"
         | 
| 5 | 
            -
             | 
| 6 | 
            -
            module Rbitter
         | 
| 7 | 
            -
              class DLThread
         | 
| 8 | 
            -
                def initialize(dlfolder, large_flag)
         | 
| 9 | 
            -
                  @dest = dlfolder
         | 
| 10 | 
            -
                  if not File.directory?(dlfolder)
         | 
| 11 | 
            -
                    warn "[dlthread] Given download location is not available for downloading."
         | 
| 12 | 
            -
                    warn "[dlthread] Fallback to current directory."
         | 
| 13 | 
            -
                    @dest = "./"
         | 
| 14 | 
            -
                  end
         | 
| 15 | 
            -
             | 
| 16 | 
            -
                  if large_flag.nil?
         | 
| 17 | 
            -
                    @large_image = false
         | 
| 18 | 
            -
                  else
         | 
| 19 | 
            -
                    @large_image = large_flag
         | 
| 20 | 
            -
                  end
         | 
| 21 | 
            -
             | 
| 22 | 
            -
                  @pool = Array.new
         | 
| 23 | 
            -
                end
         | 
| 24 | 
            -
             | 
| 25 | 
            -
                def <<(url_array)
         | 
| 26 | 
            -
                  download_task = Thread.new {
         | 
| 27 | 
            -
                    url_array.each { |url|
         | 
| 28 | 
            -
                      uri = URI.parse(@large_image ? url + ":large" : url) 
         | 
| 29 | 
            -
                      ssl = uri.scheme.downcase == 'https'
         | 
| 30 | 
            -
             | 
| 31 | 
            -
                      Net::HTTP.start(uri.host, uri.port, :use_ssl => ssl) { |h|
         | 
| 32 | 
            -
                        req = Net::HTTP::Get.new uri.request_uri
         | 
| 33 | 
            -
                        h.request(req) { |res|
         | 
| 34 | 
            -
                          case res
         | 
| 35 | 
            -
                          when Net::HTTPOK
         | 
| 36 | 
            -
                            fname = File.basename(url)
         | 
| 37 | 
            -
             | 
| 38 | 
            -
                            puts "[fetch] remote: #{uri.path} => local: #{fname}"
         | 
| 39 | 
            -
                            open(File.join(@dest, fname), "wb") { |file|
         | 
| 40 | 
            -
                              res.read_body { |chunk| file.write(chunk) }
         | 
| 41 | 
            -
                            }
         | 
| 42 | 
            -
                          end
         | 
| 43 | 
            -
                        }
         | 
| 44 | 
            -
                      }
         | 
| 45 | 
            -
                    }
         | 
| 46 | 
            -
                  }
         | 
| 47 | 
            -
             | 
| 48 | 
            -
                  @pool.push download_task
         | 
| 49 | 
            -
                end
         | 
| 50 | 
            -
             | 
| 51 | 
            -
                def job_cleanup
         | 
| 52 | 
            -
                  until @pool.empty?
         | 
| 53 | 
            -
                    dlthrd = @pool.shift
         | 
| 54 | 
            -
             | 
| 55 | 
            -
                    if dlthrd.alive?
         | 
| 56 | 
            -
                      puts "[dlthread] Thread forceful cleaning up [remains: #{@pool.length}]"
         | 
| 57 | 
            -
                      dlthrd.terminate
         | 
| 58 | 
            -
                      dlthrd.join
         | 
| 59 | 
            -
                    end
         | 
| 60 | 
            -
                  end
         | 
| 61 | 
            -
                end
         | 
| 62 | 
            -
              end
         | 
| 1 | 
            +
            # encoding: utf-8
         | 
| 2 | 
            +
             | 
| 3 | 
            +
            require "net/http"
         | 
| 4 | 
            +
            require "openssl"
         | 
| 5 | 
            +
             | 
| 6 | 
            +
            module Rbitter
         | 
| 7 | 
            +
              class DLThread
         | 
| 8 | 
            +
                def initialize(dlfolder, large_flag)
         | 
| 9 | 
            +
                  @dest = dlfolder
         | 
| 10 | 
            +
                  if not File.directory?(dlfolder)
         | 
| 11 | 
            +
                    warn "[dlthread] Given download location is not available for downloading."
         | 
| 12 | 
            +
                    warn "[dlthread] Fallback to current directory."
         | 
| 13 | 
            +
                    @dest = "./"
         | 
| 14 | 
            +
                  end
         | 
| 15 | 
            +
             | 
| 16 | 
            +
                  if large_flag.nil?
         | 
| 17 | 
            +
                    @large_image = false
         | 
| 18 | 
            +
                  else
         | 
| 19 | 
            +
                    @large_image = large_flag
         | 
| 20 | 
            +
                  end
         | 
| 21 | 
            +
             | 
| 22 | 
            +
                  @pool = Array.new
         | 
| 23 | 
            +
                end
         | 
| 24 | 
            +
             | 
| 25 | 
            +
                def <<(url_array)
         | 
| 26 | 
            +
                  download_task = Thread.new {
         | 
| 27 | 
            +
                    url_array.each { |url|
         | 
| 28 | 
            +
                      uri = URI.parse(@large_image ? url + ":large" : url) 
         | 
| 29 | 
            +
                      ssl = uri.scheme.downcase == 'https'
         | 
| 30 | 
            +
             | 
| 31 | 
            +
                      Net::HTTP.start(uri.host, uri.port, :use_ssl => ssl) { |h|
         | 
| 32 | 
            +
                        req = Net::HTTP::Get.new uri.request_uri
         | 
| 33 | 
            +
                        h.request(req) { |res|
         | 
| 34 | 
            +
                          case res
         | 
| 35 | 
            +
                          when Net::HTTPOK
         | 
| 36 | 
            +
                            fname = File.basename(url)
         | 
| 37 | 
            +
             | 
| 38 | 
            +
                            puts "[fetch] remote: #{uri.path} => local: #{fname}"
         | 
| 39 | 
            +
                            open(File.join(@dest, fname), "wb") { |file|
         | 
| 40 | 
            +
                              res.read_body { |chunk| file.write(chunk) }
         | 
| 41 | 
            +
                            }
         | 
| 42 | 
            +
                          end
         | 
| 43 | 
            +
                        }
         | 
| 44 | 
            +
                      }
         | 
| 45 | 
            +
                    }
         | 
| 46 | 
            +
                  }
         | 
| 47 | 
            +
             | 
| 48 | 
            +
                  @pool.push download_task
         | 
| 49 | 
            +
                end
         | 
| 50 | 
            +
             | 
| 51 | 
            +
                def job_cleanup
         | 
| 52 | 
            +
                  until @pool.empty?
         | 
| 53 | 
            +
                    dlthrd = @pool.shift
         | 
| 54 | 
            +
             | 
| 55 | 
            +
                    if dlthrd.alive?
         | 
| 56 | 
            +
                      puts "[dlthread] Thread forceful cleaning up [remains: #{@pool.length}]"
         | 
| 57 | 
            +
                      dlthrd.terminate
         | 
| 58 | 
            +
                      dlthrd.join
         | 
| 59 | 
            +
                    end
         | 
| 60 | 
            +
                  end
         | 
| 61 | 
            +
                end
         | 
| 62 | 
            +
              end
         | 
| 63 63 | 
             
            end
         | 
    
        data/lib/rbitter/env.rb
    CHANGED
    
    | @@ -1,116 +1,116 @@ | |
| 1 | 
            -
            # encoding: utf-8
         | 
| 2 | 
            -
             | 
| 3 | 
            -
            require "json"
         | 
| 4 | 
            -
            require "rbitter/default/config_json"
         | 
| 5 | 
            -
             | 
| 6 | 
            -
            module Rbitter
         | 
| 7 | 
            -
              @@env = Hash.new
         | 
| 8 | 
            -
             | 
| 9 | 
            -
              class ConfigFileError < StandardError; end
         | 
| 10 | 
            -
              class MissingFieldError < StandardError; end
         | 
| 11 | 
            -
             | 
| 12 | 
            -
              def self.[](k)
         | 
| 13 | 
            -
                @@env[k]
         | 
| 14 | 
            -
              end
         | 
| 15 | 
            -
             | 
| 16 | 
            -
              module_function
         | 
| 17 | 
            -
              def env
         | 
| 18 | 
            -
                @@env
         | 
| 19 | 
            -
              end
         | 
| 20 | 
            -
             | 
| 21 | 
            -
              def env_reset
         | 
| 22 | 
            -
                @@env.clear
         | 
| 23 | 
            -
              end
         | 
| 24 | 
            -
             | 
| 25 | 
            -
              def env_listfields hash
         | 
| 26 | 
            -
                path_stack = ['']
         | 
| 27 | 
            -
                generated = []
         | 
| 28 | 
            -
             | 
| 29 | 
            -
                until path_stack.empty?
         | 
| 30 | 
            -
                  path = path_stack.pop
         | 
| 31 | 
            -
             | 
| 32 | 
            -
                  if path == ''
         | 
| 33 | 
            -
                    o = hash
         | 
| 34 | 
            -
                  else
         | 
| 35 | 
            -
                    nodes = path.strip.split('->')
         | 
| 36 | 
            -
                    o = hash
         | 
| 37 | 
            -
                    until nodes.empty?
         | 
| 38 | 
            -
                      o = o[nodes.shift]
         | 
| 39 | 
            -
                    end
         | 
| 40 | 
            -
                  end
         | 
| 41 | 
            -
             | 
| 42 | 
            -
                  o.each_key { |k|
         | 
| 43 | 
            -
                    if o[k].is_a?(Hash)
         | 
| 44 | 
            -
                      path_stack << "#{k}" if path.empty?
         | 
| 45 | 
            -
                      path_stack << path + "->#{k}" unless path.empty?
         | 
| 46 | 
            -
                    else
         | 
| 47 | 
            -
                      generated << "#{k}" if path.empty?
         | 
| 48 | 
            -
                      generated << path + "->#{k}" unless path.empty?
         | 
| 49 | 
            -
                    end
         | 
| 50 | 
            -
                  }
         | 
| 51 | 
            -
                end
         | 
| 52 | 
            -
             | 
| 53 | 
            -
                generated
         | 
| 54 | 
            -
              end
         | 
| 55 | 
            -
             | 
| 56 | 
            -
              def env_valid?
         | 
| 57 | 
            -
                defaults = env_listfields(JSON.parse(DEFAULT_CONFIG_JSON))
         | 
| 58 | 
            -
                currents = env_listfields(@@env)
         | 
| 59 | 
            -
                not_errored = true
         | 
| 60 | 
            -
             | 
| 61 | 
            -
                # Cross checking (2 phases)
         | 
| 62 | 
            -
                # In current exists, default does not: redundant configuration
         | 
| 63 | 
            -
                # Level: warning since it is not utilized at all.
         | 
| 64 | 
            -
                currents.each { |conf|
         | 
| 65 | 
            -
                  unless defaults.include?(conf)
         | 
| 66 | 
            -
                    warn "[config.json] Unused config: #{conf}. You can safely remove it."
         | 
| 67 | 
            -
                  end
         | 
| 68 | 
            -
                }
         | 
| 69 | 
            -
             | 
| 70 | 
            -
                # In default exists, current does not: missing configuration
         | 
| 71 | 
            -
                # Level: error and program should stop. (return false for this)
         | 
| 72 | 
            -
                defaults.each { |conf|
         | 
| 73 | 
            -
                  unless currents.include?(conf)
         | 
| 74 | 
            -
                    warn "[config.json] Config not found: #{conf}. Invalid configuration!"
         | 
| 75 | 
            -
                    not_errored = false
         | 
| 76 | 
            -
                  end
         | 
| 77 | 
            -
                }
         | 
| 78 | 
            -
                not_errored
         | 
| 79 | 
            -
              end
         | 
| 80 | 
            -
             | 
| 81 | 
            -
              def config_initialize json_path=nil
         | 
| 82 | 
            -
                env_reset
         | 
| 83 | 
            -
             | 
| 84 | 
            -
                unless json_path.nil?
         | 
| 85 | 
            -
                  begin
         | 
| 86 | 
            -
                    open(json_path, 'r') { |file|
         | 
| 87 | 
            -
                      @@env = JSON.parse(file.read)
         | 
| 88 | 
            -
                    }
         | 
| 89 | 
            -
             | 
| 90 | 
            -
                    return @@env if env_valid?
         | 
| 91 | 
            -
                    fail StandardError, "Invalid configuration"
         | 
| 92 | 
            -
                  rescue => e
         | 
| 93 | 
            -
                    fail ConfigFileError, "Load Failure (#{json_path}): #{e.to_s}"
         | 
| 94 | 
            -
                  end
         | 
| 95 | 
            -
                end
         | 
| 96 | 
            -
             | 
| 97 | 
            -
                # Configuration default location
         | 
| 98 | 
            -
                # 1. (current_dir)/config.json
         | 
| 99 | 
            -
                # 2. (current_dir)/.rbitter/config.json
         | 
| 100 | 
            -
                locations = ["config.json", ".rbitter/config.json"]
         | 
| 101 | 
            -
                locations.collect! { |base| File.join(Dir.pwd, base) }
         | 
| 102 | 
            -
             | 
| 103 | 
            -
                for location in locations
         | 
| 104 | 
            -
                  next unless File.file?(location)
         | 
| 105 | 
            -
                  open(location, 'r') { |file|
         | 
| 106 | 
            -
                    @@env = JSON.parse(file.read)
         | 
| 107 | 
            -
                  }
         | 
| 108 | 
            -
                  break unless @@env.empty?
         | 
| 109 | 
            -
                end
         | 
| 110 | 
            -
             | 
| 111 | 
            -
                fail ConfigFileError, "No config.json on #{locations.join(' or ')}" if @@env.empty?
         | 
| 112 | 
            -
                fail ConfigFileError, "Configuration outdated. Please see above messages to update it" if not env_valid?
         | 
| 113 | 
            -
             | 
| 114 | 
            -
                puts "[config.json] Loaded configuration is valid. good to go!"
         | 
| 115 | 
            -
              end
         | 
| 116 | 
            -
            end
         | 
| 1 | 
            +
            # encoding: utf-8
         | 
| 2 | 
            +
             | 
| 3 | 
            +
            require "json"
         | 
| 4 | 
            +
            require "rbitter/default/config_json"
         | 
| 5 | 
            +
             | 
| 6 | 
            +
            module Rbitter
         | 
| 7 | 
            +
              @@env = Hash.new
         | 
| 8 | 
            +
             | 
| 9 | 
            +
              class ConfigFileError < StandardError; end
         | 
| 10 | 
            +
              class MissingFieldError < StandardError; end
         | 
| 11 | 
            +
             | 
| 12 | 
            +
              def self.[](k)
         | 
| 13 | 
            +
                @@env[k]
         | 
| 14 | 
            +
              end
         | 
| 15 | 
            +
             | 
| 16 | 
            +
              module_function
         | 
| 17 | 
            +
              def env
         | 
| 18 | 
            +
                @@env
         | 
| 19 | 
            +
              end
         | 
| 20 | 
            +
             | 
| 21 | 
            +
              def env_reset
         | 
| 22 | 
            +
                @@env.clear
         | 
| 23 | 
            +
              end
         | 
| 24 | 
            +
             | 
| 25 | 
            +
              def env_listfields hash
         | 
| 26 | 
            +
                path_stack = ['']
         | 
| 27 | 
            +
                generated = []
         | 
| 28 | 
            +
             | 
| 29 | 
            +
                until path_stack.empty?
         | 
| 30 | 
            +
                  path = path_stack.pop
         | 
| 31 | 
            +
             | 
| 32 | 
            +
                  if path == ''
         | 
| 33 | 
            +
                    o = hash
         | 
| 34 | 
            +
                  else
         | 
| 35 | 
            +
                    nodes = path.strip.split('->')
         | 
| 36 | 
            +
                    o = hash
         | 
| 37 | 
            +
                    until nodes.empty?
         | 
| 38 | 
            +
                      o = o[nodes.shift]
         | 
| 39 | 
            +
                    end
         | 
| 40 | 
            +
                  end
         | 
| 41 | 
            +
             | 
| 42 | 
            +
                  o.each_key { |k|
         | 
| 43 | 
            +
                    if o[k].is_a?(Hash)
         | 
| 44 | 
            +
                      path_stack << "#{k}" if path.empty?
         | 
| 45 | 
            +
                      path_stack << path + "->#{k}" unless path.empty?
         | 
| 46 | 
            +
                    else
         | 
| 47 | 
            +
                      generated << "#{k}" if path.empty?
         | 
| 48 | 
            +
                      generated << path + "->#{k}" unless path.empty?
         | 
| 49 | 
            +
                    end
         | 
| 50 | 
            +
                  }
         | 
| 51 | 
            +
                end
         | 
| 52 | 
            +
             | 
| 53 | 
            +
                generated
         | 
| 54 | 
            +
              end
         | 
| 55 | 
            +
             | 
| 56 | 
            +
              def env_valid?
         | 
| 57 | 
            +
                defaults = env_listfields(JSON.parse(DEFAULT_CONFIG_JSON))
         | 
| 58 | 
            +
                currents = env_listfields(@@env)
         | 
| 59 | 
            +
                not_errored = true
         | 
| 60 | 
            +
             | 
| 61 | 
            +
                # Cross checking (2 phases)
         | 
| 62 | 
            +
                # In current exists, default does not: redundant configuration
         | 
| 63 | 
            +
                # Level: warning since it is not utilized at all.
         | 
| 64 | 
            +
                currents.each { |conf|
         | 
| 65 | 
            +
                  unless defaults.include?(conf)
         | 
| 66 | 
            +
                    warn "[config.json] Unused config: #{conf}. You can safely remove it."
         | 
| 67 | 
            +
                  end
         | 
| 68 | 
            +
                }
         | 
| 69 | 
            +
             | 
| 70 | 
            +
                # In default exists, current does not: missing configuration
         | 
| 71 | 
            +
                # Level: error and program should stop. (return false for this)
         | 
| 72 | 
            +
                defaults.each { |conf|
         | 
| 73 | 
            +
                  unless currents.include?(conf)
         | 
| 74 | 
            +
                    warn "[config.json] Config not found: #{conf}. Invalid configuration!"
         | 
| 75 | 
            +
                    not_errored = false
         | 
| 76 | 
            +
                  end
         | 
| 77 | 
            +
                }
         | 
| 78 | 
            +
                not_errored
         | 
| 79 | 
            +
              end
         | 
| 80 | 
            +
             | 
| 81 | 
            +
              def config_initialize json_path=nil
         | 
| 82 | 
            +
                env_reset
         | 
| 83 | 
            +
             | 
| 84 | 
            +
                unless json_path.nil?
         | 
| 85 | 
            +
                  begin
         | 
| 86 | 
            +
                    open(json_path, 'r') { |file|
         | 
| 87 | 
            +
                      @@env = JSON.parse(file.read)
         | 
| 88 | 
            +
                    }
         | 
| 89 | 
            +
             | 
| 90 | 
            +
                    return @@env if env_valid?
         | 
| 91 | 
            +
                    fail StandardError, "Invalid configuration"
         | 
| 92 | 
            +
                  rescue => e
         | 
| 93 | 
            +
                    fail ConfigFileError, "Load Failure (#{json_path}): #{e.to_s}"
         | 
| 94 | 
            +
                  end
         | 
| 95 | 
            +
                end
         | 
| 96 | 
            +
             | 
| 97 | 
            +
                # Configuration default location
         | 
| 98 | 
            +
                # 1. (current_dir)/config.json
         | 
| 99 | 
            +
                # 2. (current_dir)/.rbitter/config.json
         | 
| 100 | 
            +
                locations = ["config.json", ".rbitter/config.json"]
         | 
| 101 | 
            +
                locations.collect! { |base| File.join(Dir.pwd, base) }
         | 
| 102 | 
            +
             | 
| 103 | 
            +
                for location in locations
         | 
| 104 | 
            +
                  next unless File.file?(location)
         | 
| 105 | 
            +
                  open(location, 'r') { |file|
         | 
| 106 | 
            +
                    @@env = JSON.parse(file.read)
         | 
| 107 | 
            +
                  }
         | 
| 108 | 
            +
                  break unless @@env.empty?
         | 
| 109 | 
            +
                end
         | 
| 110 | 
            +
             | 
| 111 | 
            +
                fail ConfigFileError, "No config.json on #{locations.join(' or ')}" if @@env.empty?
         | 
| 112 | 
            +
                fail ConfigFileError, "Configuration outdated. Please see above messages to update it" if not env_valid?
         | 
| 113 | 
            +
             | 
| 114 | 
            +
                puts "[config.json] Loaded configuration is valid. good to go!"
         | 
| 115 | 
            +
              end
         | 
| 116 | 
            +
            end
         | 
| @@ -1,8 +1,8 @@ | |
| 1 | 
            -
            # encoding: utf-8
         | 
| 2 | 
            -
            require 'socket'
         | 
| 3 | 
            -
             | 
| 4 | 
            -
            class Socket
         | 
| 5 | 
            -
              def self.ip_address_list
         | 
| 6 | 
            -
                fail NotImplementedError
         | 
| 7 | 
            -
              end
         | 
| 8 | 
            -
            end
         | 
| 1 | 
            +
            # encoding: utf-8
         | 
| 2 | 
            +
            require 'socket'
         | 
| 3 | 
            +
             | 
| 4 | 
            +
            class Socket
         | 
| 5 | 
            +
              def self.ip_address_list
         | 
| 6 | 
            +
                fail NotImplementedError
         | 
| 7 | 
            +
              end
         | 
| 8 | 
            +
            end
         |