rbitter 0.2.0-java → 0.2.1-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +23 -22
  3. data/.rspec +2 -2
  4. data/.travis.yml +15 -15
  5. data/Gemfile +12 -12
  6. data/LICENSE.txt +22 -22
  7. data/Rakefile +8 -8
  8. data/bin/rbitter +20 -20
  9. data/lib/rbitter/arcserver.rb +169 -165
  10. data/lib/rbitter/console.rb +93 -93
  11. data/lib/rbitter/default/config_json.rb +41 -41
  12. data/lib/rbitter/dlthread.rb +62 -62
  13. data/lib/rbitter/env.rb +116 -116
  14. data/lib/rbitter/override/gems/rubysl-socket/socket.rb +8 -8
  15. data/lib/rbitter/override/gems/twitter/connection.rb +45 -45
  16. data/lib/rbitter/override.rb +47 -47
  17. data/lib/rbitter/progress.rb +23 -0
  18. data/lib/rbitter/records.rb +127 -127
  19. data/lib/rbitter/records_migrate/20150327_add_index.rb +11 -11
  20. data/lib/rbitter/records_migrate/20150504_add_replyto_column.rb +11 -11
  21. data/lib/rbitter/streaming.rb +104 -104
  22. data/lib/rbitter/version.rb +20 -20
  23. data/lib/rbitter/xmlrpc.rb +3 -3
  24. data/lib/rbitter/xmlrpcd/base.rb +24 -24
  25. data/lib/rbitter/xmlrpcd/rpchandles.rb +11 -11
  26. data/lib/rbitter/xmlrpcd/xmlrpc_auth_server.rb +82 -82
  27. data/lib/rbitter/xmlrpcd/xmlrpcd.rb +69 -69
  28. data/lib/rbitter.rb +62 -62
  29. data/rbitter.gemspec +46 -46
  30. data/spec/config/default.json +32 -32
  31. data/spec/rbitter/arcserver_spec.rb +30 -30
  32. data/spec/rbitter/console_spec.rb +9 -9
  33. data/spec/rbitter/default/config_json_spec.rb +3 -3
  34. data/spec/rbitter/dlthread_spec.rb +8 -8
  35. data/spec/rbitter/env_spec.rb +76 -76
  36. data/spec/rbitter/override/gems/twitter/connection_spec.rb +8 -8
  37. data/spec/rbitter/progress_spec.rb +1 -0
  38. data/spec/rbitter/records_spec.rb +13 -13
  39. data/spec/rbitter/streaming_spec.rb +9 -9
  40. data/spec/rbitter/version_spec.rb +8 -8
  41. data/spec/rbitter/xmlrpc_spec.rb +8 -8
  42. data/spec/rbitter/xmlrpcd/base_spec.rb +29 -29
  43. data/spec/rbitter/xmlrpcd/rpchandles_spec.rb +10 -10
  44. data/spec/rbitter/xmlrpcd/xmlrpc_auth_server_spec.rb +8 -8
  45. data/spec/rbitter/xmlrpcd/xmlrpcd_spec.rb +9 -9
  46. data/spec/rbitter_spec.rb +38 -38
  47. data/spec/spec_helper.rb +39 -39
  48. metadata +6 -3
@@ -1,93 +1,93 @@
1
- # encoding: utf-8
2
- #
3
- # Rbitter Archive Access console (irb)
4
-
5
- require "xmlrpc/client"
6
- require "rbitter/version"
7
- require "ripl"
8
-
9
- module Rbitter
10
- class Console
11
- def initialize
12
- puts "Rbitter console #{Rbitter::VERSION}"
13
- help
14
- end
15
-
16
- def help
17
- puts "Predefined methods:"
18
- puts "ar - shortcut to call Rbitter::Record"
19
- puts "connect_ar - Prepare Rbitter::Record to be ready"
20
- puts "csv_backup - export Rbitter::Record into comma-separated values"
21
- puts "help - to show this message again"
22
- puts "xmlrpc - send xmlrpc command to destination"
23
- puts "xmlrpc_dest - set destination for xmlrpc command"
24
- puts "^D, 'exit' to exit from here"
25
- end
26
-
27
- def connect_ar
28
- ARSupport.connect_database
29
- puts "Rbitter::Record is ready."
30
- end
31
-
32
- def csv_backup *args
33
- if args.length < 0
34
- puts "Usage: csv_backup('filename.csv')"
35
- puts "Estimated running time depends on system environment"
36
- else
37
- ARSupport.export_to_csv(args[0])
38
- end
39
- end
40
-
41
- def ar
42
- Rbitter::Record
43
- end
44
-
45
- def exit
46
- Kernel.exit(0)
47
- end
48
-
49
- def xmlrpc_dest args={}
50
- if args.empty?
51
- puts "Usage: xmlrpc_dest({ :rpchost => '', :rpcpath => '', :rpcport => 1400,"
52
- puts " :xmlrpc_auth_id => '', xmlrpc_auth_password => '' })"
53
- end
54
-
55
- @rpchost = args.fetch(:rpchost) { "127.0.0.1" }
56
- @rpcpath = args.fetch(:rpcpath) { "/" }
57
- @rpcport = args.fetch(:rpcport) { 1400 }
58
-
59
- cl = XMLRPC::Client.new(@rpchost, @rpcpath, @rpcport)
60
- @xmlrpc_cookie = "auth_key=" + cl.call('rbitter.auth',
61
- args.fetch(:xmlrpc_auth_id) { Rbitter.env['xmlrpc']['auth'][0] },
62
- args.fetch(:xmlrpc_auth_password) { Rbitter.env['xmlrpc']['auth'][1] } )
63
-
64
- if @xmlrpc_cookie != "auth_key="
65
- puts "Authentication completed"
66
- else
67
- puts "Authentication failed"
68
- end
69
- end
70
-
71
- def xmlrpc *args
72
- if args.empty?
73
- puts "Usage: xmlrpc(command, [params in Array])"
74
- puts "Ex) xmlrpc(\'rbitter.echo\',' [\"Hello World!\"])"
75
- puts "Please configure XMLRPC destination with xmlrpc_dest method"
76
- return false
77
- end
78
-
79
- cl = XMLRPC::Client.new(@rpchost, @rpcpath, @rpcport, @xmlrpc_cookie)
80
-
81
- if args.length <= 1 or args[1].nil?
82
- cl.call(args[0])
83
- else
84
- cl.call(args[0], *args[1])
85
- end
86
- end
87
-
88
- def start
89
- Ripl.start :binding => binding
90
- end
91
- end
92
- end
93
-
1
+ # encoding: utf-8
2
+ #
3
+ # Rbitter Archive Access console (irb)
4
+
5
+ require "xmlrpc/client"
6
+ require "rbitter/version"
7
+ require "ripl"
8
+
9
+ module Rbitter
10
+ class Console
11
+ def initialize
12
+ puts "Rbitter console #{Rbitter::VERSION}"
13
+ help
14
+ end
15
+
16
+ def help
17
+ puts "Predefined methods:"
18
+ puts "ar - shortcut to call Rbitter::Record"
19
+ puts "connect_ar - Prepare Rbitter::Record to be ready"
20
+ puts "csv_backup - export Rbitter::Record into comma-separated values"
21
+ puts "help - to show this message again"
22
+ puts "xmlrpc - send xmlrpc command to destination"
23
+ puts "xmlrpc_dest - set destination for xmlrpc command"
24
+ puts "^D, 'exit' to exit from here"
25
+ end
26
+
27
+ def connect_ar
28
+ ARSupport.connect_database
29
+ puts "Rbitter::Record is ready."
30
+ end
31
+
32
+ def csv_backup *args
33
+ if args.length < 0
34
+ puts "Usage: csv_backup('filename.csv')"
35
+ puts "Estimated running time depends on system environment"
36
+ else
37
+ ARSupport.export_to_csv(args[0])
38
+ end
39
+ end
40
+
41
+ def ar
42
+ Rbitter::Record
43
+ end
44
+
45
+ def exit
46
+ Kernel.exit(0)
47
+ end
48
+
49
+ def xmlrpc_dest args={}
50
+ if args.empty?
51
+ puts "Usage: xmlrpc_dest({ :rpchost => '', :rpcpath => '', :rpcport => 1400,"
52
+ puts " :xmlrpc_auth_id => '', xmlrpc_auth_password => '' })"
53
+ end
54
+
55
+ @rpchost = args.fetch(:rpchost) { "127.0.0.1" }
56
+ @rpcpath = args.fetch(:rpcpath) { "/" }
57
+ @rpcport = args.fetch(:rpcport) { 1400 }
58
+
59
+ cl = XMLRPC::Client.new(@rpchost, @rpcpath, @rpcport)
60
+ @xmlrpc_cookie = "auth_key=" + cl.call('rbitter.auth',
61
+ args.fetch(:xmlrpc_auth_id) { Rbitter.env['xmlrpc']['auth'][0] },
62
+ args.fetch(:xmlrpc_auth_password) { Rbitter.env['xmlrpc']['auth'][1] } )
63
+
64
+ if @xmlrpc_cookie != "auth_key="
65
+ puts "Authentication completed"
66
+ else
67
+ puts "Authentication failed"
68
+ end
69
+ end
70
+
71
+ def xmlrpc *args
72
+ if args.empty?
73
+ puts "Usage: xmlrpc(command, [params in Array])"
74
+ puts "Ex) xmlrpc(\'rbitter.echo\',' [\"Hello World!\"])"
75
+ puts "Please configure XMLRPC destination with xmlrpc_dest method"
76
+ return false
77
+ end
78
+
79
+ cl = XMLRPC::Client.new(@rpchost, @rpcpath, @rpcport, @xmlrpc_cookie)
80
+
81
+ if args.length <= 1 or args[1].nil?
82
+ cl.call(args[0])
83
+ else
84
+ cl.call(args[0], *args[1])
85
+ end
86
+ end
87
+
88
+ def start
89
+ Ripl.start :binding => binding
90
+ end
91
+ end
92
+ end
93
+
@@ -1,41 +1,41 @@
1
- module Rbitter
2
- DEFAULT_CONFIG_JSON = <<-ENDOFJSON
3
- {
4
- "twitter": {
5
- "consumer_key": "",
6
- "consumer_secret": "",
7
- "access_token": "",
8
- "access_token_secret": "",
9
- "connection": {
10
- "reconnect": true,
11
- "timeout_secs": 5
12
- }
13
- },
14
- "activerecord": "sqlite3",
15
- "sqlite3": {
16
- "dbfile": "rbitter.sqlite"
17
- },
18
- "mysql2": {
19
- "host": "localhost",
20
- "port": 3306,
21
- "dbname": "archive",
22
- "username": "",
23
- "password": ""
24
- },
25
- "media_downloader": {
26
- "large_image": true,
27
- "download_dir": "imgs/"
28
- },
29
- "xmlrpc": {
30
- "enable": true,
31
- "bind_host": "0.0.0.0",
32
- "bind_port": 1400,
33
- "auth": {
34
- "username": "username",
35
- "password": "password"
36
- },
37
- "handles": ["/path/to/handles"]
38
- }
39
- }
40
- ENDOFJSON
41
- end
1
+ module Rbitter
2
+ DEFAULT_CONFIG_JSON = <<-ENDOFJSON
3
+ {
4
+ "twitter": {
5
+ "consumer_key": "",
6
+ "consumer_secret": "",
7
+ "access_token": "",
8
+ "access_token_secret": "",
9
+ "connection": {
10
+ "reconnect": true,
11
+ "timeout_secs": 5
12
+ }
13
+ },
14
+ "activerecord": "sqlite3",
15
+ "sqlite3": {
16
+ "dbfile": "rbitter.sqlite"
17
+ },
18
+ "mysql2": {
19
+ "host": "localhost",
20
+ "port": 3306,
21
+ "dbname": "archive",
22
+ "username": "",
23
+ "password": ""
24
+ },
25
+ "media_downloader": {
26
+ "large_image": true,
27
+ "download_dir": "imgs/"
28
+ },
29
+ "xmlrpc": {
30
+ "enable": true,
31
+ "bind_host": "0.0.0.0",
32
+ "bind_port": 1400,
33
+ "auth": {
34
+ "username": "username",
35
+ "password": "password"
36
+ },
37
+ "handles": ["/path/to/handles"]
38
+ }
39
+ }
40
+ ENDOFJSON
41
+ end
@@ -1,63 +1,63 @@
1
- # encoding: utf-8
2
-
3
- require "net/http"
4
- require "openssl"
5
-
6
- module Rbitter
7
- class DLThread
8
- def initialize(dlfolder, large_flag)
9
- @dest = dlfolder
10
- if not File.directory?(dlfolder)
11
- warn "[dlthread] Given download location is not available for downloading."
12
- warn "[dlthread] Fallback to current directory."
13
- @dest = "./"
14
- end
15
-
16
- if large_flag.nil?
17
- @large_image = false
18
- else
19
- @large_image = large_flag
20
- end
21
-
22
- @pool = Array.new
23
- end
24
-
25
- def <<(url_array)
26
- download_task = Thread.new {
27
- url_array.each { |url|
28
- uri = URI.parse(@large_image ? url + ":large" : url)
29
- ssl = uri.scheme.downcase == 'https'
30
-
31
- Net::HTTP.start(uri.host, uri.port, :use_ssl => ssl) { |h|
32
- req = Net::HTTP::Get.new uri.request_uri
33
- h.request(req) { |res|
34
- case res
35
- when Net::HTTPOK
36
- fname = File.basename(url)
37
-
38
- puts "[fetch] remote: #{uri.path} => local: #{fname}"
39
- open(File.join(@dest, fname), "wb") { |file|
40
- res.read_body { |chunk| file.write(chunk) }
41
- }
42
- end
43
- }
44
- }
45
- }
46
- }
47
-
48
- @pool.push download_task
49
- end
50
-
51
- def job_cleanup
52
- until @pool.empty?
53
- dlthrd = @pool.shift
54
-
55
- if dlthrd.alive?
56
- puts "[dlthread] Thread forceful cleaning up [remains: #{@pool.length}]"
57
- dlthrd.terminate
58
- dlthrd.join
59
- end
60
- end
61
- end
62
- end
1
+ # encoding: utf-8
2
+
3
+ require "net/http"
4
+ require "openssl"
5
+
6
+ module Rbitter
7
+ class DLThread
8
+ def initialize(dlfolder, large_flag)
9
+ @dest = dlfolder
10
+ if not File.directory?(dlfolder)
11
+ warn "[dlthread] Given download location is not available for downloading."
12
+ warn "[dlthread] Fallback to current directory."
13
+ @dest = "./"
14
+ end
15
+
16
+ if large_flag.nil?
17
+ @large_image = false
18
+ else
19
+ @large_image = large_flag
20
+ end
21
+
22
+ @pool = Array.new
23
+ end
24
+
25
+ def <<(url_array)
26
+ download_task = Thread.new {
27
+ url_array.each { |url|
28
+ uri = URI.parse(@large_image ? url + ":large" : url)
29
+ ssl = uri.scheme.downcase == 'https'
30
+
31
+ Net::HTTP.start(uri.host, uri.port, :use_ssl => ssl) { |h|
32
+ req = Net::HTTP::Get.new uri.request_uri
33
+ h.request(req) { |res|
34
+ case res
35
+ when Net::HTTPOK
36
+ fname = File.basename(url)
37
+
38
+ puts "[fetch] remote: #{uri.path} => local: #{fname}"
39
+ open(File.join(@dest, fname), "wb") { |file|
40
+ res.read_body { |chunk| file.write(chunk) }
41
+ }
42
+ end
43
+ }
44
+ }
45
+ }
46
+ }
47
+
48
+ @pool.push download_task
49
+ end
50
+
51
+ def job_cleanup
52
+ until @pool.empty?
53
+ dlthrd = @pool.shift
54
+
55
+ if dlthrd.alive?
56
+ puts "[dlthread] Thread forceful cleaning up [remains: #{@pool.length}]"
57
+ dlthrd.terminate
58
+ dlthrd.join
59
+ end
60
+ end
61
+ end
62
+ end
63
63
  end
data/lib/rbitter/env.rb CHANGED
@@ -1,116 +1,116 @@
1
- # encoding: utf-8
2
-
3
- require "json"
4
- require "rbitter/default/config_json"
5
-
6
- module Rbitter
7
- @@env = Hash.new
8
-
9
- class ConfigFileError < StandardError; end
10
- class MissingFieldError < StandardError; end
11
-
12
- def self.[](k)
13
- @@env[k]
14
- end
15
-
16
- module_function
17
- def env
18
- @@env
19
- end
20
-
21
- def env_reset
22
- @@env.clear
23
- end
24
-
25
- def env_listfields hash
26
- path_stack = ['']
27
- generated = []
28
-
29
- until path_stack.empty?
30
- path = path_stack.pop
31
-
32
- if path == ''
33
- o = hash
34
- else
35
- nodes = path.strip.split('->')
36
- o = hash
37
- until nodes.empty?
38
- o = o[nodes.shift]
39
- end
40
- end
41
-
42
- o.each_key { |k|
43
- if o[k].is_a?(Hash)
44
- path_stack << "#{k}" if path.empty?
45
- path_stack << path + "->#{k}" unless path.empty?
46
- else
47
- generated << "#{k}" if path.empty?
48
- generated << path + "->#{k}" unless path.empty?
49
- end
50
- }
51
- end
52
-
53
- generated
54
- end
55
-
56
- def env_valid?
57
- defaults = env_listfields(JSON.parse(DEFAULT_CONFIG_JSON))
58
- currents = env_listfields(@@env)
59
- not_errored = true
60
-
61
- # Cross checking (2 phases)
62
- # In current exists, default does not: redundant configuration
63
- # Level: warning since it is not utilized at all.
64
- currents.each { |conf|
65
- unless defaults.include?(conf)
66
- warn "[config.json] Unused config: #{conf}. You can safely remove it."
67
- end
68
- }
69
-
70
- # In default exists, current does not: missing configuration
71
- # Level: error and program should stop. (return false for this)
72
- defaults.each { |conf|
73
- unless currents.include?(conf)
74
- warn "[config.json] Config not found: #{conf}. Invalid configuration!"
75
- not_errored = false
76
- end
77
- }
78
- not_errored
79
- end
80
-
81
- def config_initialize json_path=nil
82
- env_reset
83
-
84
- unless json_path.nil?
85
- begin
86
- open(json_path, 'r') { |file|
87
- @@env = JSON.parse(file.read)
88
- }
89
-
90
- return @@env if env_valid?
91
- fail StandardError, "Invalid configuration"
92
- rescue => e
93
- fail ConfigFileError, "Load Failure (#{json_path}): #{e.to_s}"
94
- end
95
- end
96
-
97
- # Configuration default location
98
- # 1. (current_dir)/config.json
99
- # 2. (current_dir)/.rbitter/config.json
100
- locations = ["config.json", ".rbitter/config.json"]
101
- locations.collect! { |base| File.join(Dir.pwd, base) }
102
-
103
- for location in locations
104
- next unless File.file?(location)
105
- open(location, 'r') { |file|
106
- @@env = JSON.parse(file.read)
107
- }
108
- break unless @@env.empty?
109
- end
110
-
111
- fail ConfigFileError, "No config.json on #{locations.join(' or ')}" if @@env.empty?
112
- fail ConfigFileError, "Configuration outdated. Please see above messages to update it" if not env_valid?
113
-
114
- puts "[config.json] Loaded configuration is valid. good to go!"
115
- end
116
- end
1
+ # encoding: utf-8
2
+
3
+ require "json"
4
+ require "rbitter/default/config_json"
5
+
6
+ module Rbitter
7
+ @@env = Hash.new
8
+
9
+ class ConfigFileError < StandardError; end
10
+ class MissingFieldError < StandardError; end
11
+
12
+ def self.[](k)
13
+ @@env[k]
14
+ end
15
+
16
+ module_function
17
+ def env
18
+ @@env
19
+ end
20
+
21
+ def env_reset
22
+ @@env.clear
23
+ end
24
+
25
+ def env_listfields hash
26
+ path_stack = ['']
27
+ generated = []
28
+
29
+ until path_stack.empty?
30
+ path = path_stack.pop
31
+
32
+ if path == ''
33
+ o = hash
34
+ else
35
+ nodes = path.strip.split('->')
36
+ o = hash
37
+ until nodes.empty?
38
+ o = o[nodes.shift]
39
+ end
40
+ end
41
+
42
+ o.each_key { |k|
43
+ if o[k].is_a?(Hash)
44
+ path_stack << "#{k}" if path.empty?
45
+ path_stack << path + "->#{k}" unless path.empty?
46
+ else
47
+ generated << "#{k}" if path.empty?
48
+ generated << path + "->#{k}" unless path.empty?
49
+ end
50
+ }
51
+ end
52
+
53
+ generated
54
+ end
55
+
56
+ def env_valid?
57
+ defaults = env_listfields(JSON.parse(DEFAULT_CONFIG_JSON))
58
+ currents = env_listfields(@@env)
59
+ not_errored = true
60
+
61
+ # Cross checking (2 phases)
62
+ # In current exists, default does not: redundant configuration
63
+ # Level: warning since it is not utilized at all.
64
+ currents.each { |conf|
65
+ unless defaults.include?(conf)
66
+ warn "[config.json] Unused config: #{conf}. You can safely remove it."
67
+ end
68
+ }
69
+
70
+ # In default exists, current does not: missing configuration
71
+ # Level: error and program should stop. (return false for this)
72
+ defaults.each { |conf|
73
+ unless currents.include?(conf)
74
+ warn "[config.json] Config not found: #{conf}. Invalid configuration!"
75
+ not_errored = false
76
+ end
77
+ }
78
+ not_errored
79
+ end
80
+
81
+ def config_initialize json_path=nil
82
+ env_reset
83
+
84
+ unless json_path.nil?
85
+ begin
86
+ open(json_path, 'r') { |file|
87
+ @@env = JSON.parse(file.read)
88
+ }
89
+
90
+ return @@env if env_valid?
91
+ fail StandardError, "Invalid configuration"
92
+ rescue => e
93
+ fail ConfigFileError, "Load Failure (#{json_path}): #{e.to_s}"
94
+ end
95
+ end
96
+
97
+ # Configuration default location
98
+ # 1. (current_dir)/config.json
99
+ # 2. (current_dir)/.rbitter/config.json
100
+ locations = ["config.json", ".rbitter/config.json"]
101
+ locations.collect! { |base| File.join(Dir.pwd, base) }
102
+
103
+ for location in locations
104
+ next unless File.file?(location)
105
+ open(location, 'r') { |file|
106
+ @@env = JSON.parse(file.read)
107
+ }
108
+ break unless @@env.empty?
109
+ end
110
+
111
+ fail ConfigFileError, "No config.json on #{locations.join(' or ')}" if @@env.empty?
112
+ fail ConfigFileError, "Configuration outdated. Please see above messages to update it" if not env_valid?
113
+
114
+ puts "[config.json] Loaded configuration is valid. good to go!"
115
+ end
116
+ end
@@ -1,8 +1,8 @@
1
- # encoding: utf-8
2
- require 'socket'
3
-
4
- class Socket
5
- def self.ip_address_list
6
- fail NotImplementedError
7
- end
8
- end
1
+ # encoding: utf-8
2
+ require 'socket'
3
+
4
+ class Socket
5
+ def self.ip_address_list
6
+ fail NotImplementedError
7
+ end
8
+ end