rbitter 0.2.0-java → 0.2.1-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +23 -22
  3. data/.rspec +2 -2
  4. data/.travis.yml +15 -15
  5. data/Gemfile +12 -12
  6. data/LICENSE.txt +22 -22
  7. data/Rakefile +8 -8
  8. data/bin/rbitter +20 -20
  9. data/lib/rbitter/arcserver.rb +169 -165
  10. data/lib/rbitter/console.rb +93 -93
  11. data/lib/rbitter/default/config_json.rb +41 -41
  12. data/lib/rbitter/dlthread.rb +62 -62
  13. data/lib/rbitter/env.rb +116 -116
  14. data/lib/rbitter/override/gems/rubysl-socket/socket.rb +8 -8
  15. data/lib/rbitter/override/gems/twitter/connection.rb +45 -45
  16. data/lib/rbitter/override.rb +47 -47
  17. data/lib/rbitter/progress.rb +23 -0
  18. data/lib/rbitter/records.rb +127 -127
  19. data/lib/rbitter/records_migrate/20150327_add_index.rb +11 -11
  20. data/lib/rbitter/records_migrate/20150504_add_replyto_column.rb +11 -11
  21. data/lib/rbitter/streaming.rb +104 -104
  22. data/lib/rbitter/version.rb +20 -20
  23. data/lib/rbitter/xmlrpc.rb +3 -3
  24. data/lib/rbitter/xmlrpcd/base.rb +24 -24
  25. data/lib/rbitter/xmlrpcd/rpchandles.rb +11 -11
  26. data/lib/rbitter/xmlrpcd/xmlrpc_auth_server.rb +82 -82
  27. data/lib/rbitter/xmlrpcd/xmlrpcd.rb +69 -69
  28. data/lib/rbitter.rb +62 -62
  29. data/rbitter.gemspec +46 -46
  30. data/spec/config/default.json +32 -32
  31. data/spec/rbitter/arcserver_spec.rb +30 -30
  32. data/spec/rbitter/console_spec.rb +9 -9
  33. data/spec/rbitter/default/config_json_spec.rb +3 -3
  34. data/spec/rbitter/dlthread_spec.rb +8 -8
  35. data/spec/rbitter/env_spec.rb +76 -76
  36. data/spec/rbitter/override/gems/twitter/connection_spec.rb +8 -8
  37. data/spec/rbitter/progress_spec.rb +1 -0
  38. data/spec/rbitter/records_spec.rb +13 -13
  39. data/spec/rbitter/streaming_spec.rb +9 -9
  40. data/spec/rbitter/version_spec.rb +8 -8
  41. data/spec/rbitter/xmlrpc_spec.rb +8 -8
  42. data/spec/rbitter/xmlrpcd/base_spec.rb +29 -29
  43. data/spec/rbitter/xmlrpcd/rpchandles_spec.rb +10 -10
  44. data/spec/rbitter/xmlrpcd/xmlrpc_auth_server_spec.rb +8 -8
  45. data/spec/rbitter/xmlrpcd/xmlrpcd_spec.rb +9 -9
  46. data/spec/rbitter_spec.rb +38 -38
  47. data/spec/spec_helper.rb +39 -39
  48. metadata +6 -3
@@ -1,93 +1,93 @@
1
- # encoding: utf-8
2
- #
3
- # Rbitter Archive Access console (irb)
4
-
5
- require "xmlrpc/client"
6
- require "rbitter/version"
7
- require "ripl"
8
-
9
- module Rbitter
10
- class Console
11
- def initialize
12
- puts "Rbitter console #{Rbitter::VERSION}"
13
- help
14
- end
15
-
16
- def help
17
- puts "Predefined methods:"
18
- puts "ar - shortcut to call Rbitter::Record"
19
- puts "connect_ar - Prepare Rbitter::Record to be ready"
20
- puts "csv_backup - export Rbitter::Record into comma-separated values"
21
- puts "help - to show this message again"
22
- puts "xmlrpc - send xmlrpc command to destination"
23
- puts "xmlrpc_dest - set destination for xmlrpc command"
24
- puts "^D, 'exit' to exit from here"
25
- end
26
-
27
- def connect_ar
28
- ARSupport.connect_database
29
- puts "Rbitter::Record is ready."
30
- end
31
-
32
- def csv_backup *args
33
- if args.length < 0
34
- puts "Usage: csv_backup('filename.csv')"
35
- puts "Estimated running time depends on system environment"
36
- else
37
- ARSupport.export_to_csv(args[0])
38
- end
39
- end
40
-
41
- def ar
42
- Rbitter::Record
43
- end
44
-
45
- def exit
46
- Kernel.exit(0)
47
- end
48
-
49
- def xmlrpc_dest args={}
50
- if args.empty?
51
- puts "Usage: xmlrpc_dest({ :rpchost => '', :rpcpath => '', :rpcport => 1400,"
52
- puts " :xmlrpc_auth_id => '', xmlrpc_auth_password => '' })"
53
- end
54
-
55
- @rpchost = args.fetch(:rpchost) { "127.0.0.1" }
56
- @rpcpath = args.fetch(:rpcpath) { "/" }
57
- @rpcport = args.fetch(:rpcport) { 1400 }
58
-
59
- cl = XMLRPC::Client.new(@rpchost, @rpcpath, @rpcport)
60
- @xmlrpc_cookie = "auth_key=" + cl.call('rbitter.auth',
61
- args.fetch(:xmlrpc_auth_id) { Rbitter.env['xmlrpc']['auth'][0] },
62
- args.fetch(:xmlrpc_auth_password) { Rbitter.env['xmlrpc']['auth'][1] } )
63
-
64
- if @xmlrpc_cookie != "auth_key="
65
- puts "Authentication completed"
66
- else
67
- puts "Authentication failed"
68
- end
69
- end
70
-
71
- def xmlrpc *args
72
- if args.empty?
73
- puts "Usage: xmlrpc(command, [params in Array])"
74
- puts "Ex) xmlrpc(\'rbitter.echo\',' [\"Hello World!\"])"
75
- puts "Please configure XMLRPC destination with xmlrpc_dest method"
76
- return false
77
- end
78
-
79
- cl = XMLRPC::Client.new(@rpchost, @rpcpath, @rpcport, @xmlrpc_cookie)
80
-
81
- if args.length <= 1 or args[1].nil?
82
- cl.call(args[0])
83
- else
84
- cl.call(args[0], *args[1])
85
- end
86
- end
87
-
88
- def start
89
- Ripl.start :binding => binding
90
- end
91
- end
92
- end
93
-
1
+ # encoding: utf-8
2
+ #
3
+ # Rbitter Archive Access console (irb)
4
+
5
+ require "xmlrpc/client"
6
+ require "rbitter/version"
7
+ require "ripl"
8
+
9
+ module Rbitter
10
+ class Console
11
+ def initialize
12
+ puts "Rbitter console #{Rbitter::VERSION}"
13
+ help
14
+ end
15
+
16
+ def help
17
+ puts "Predefined methods:"
18
+ puts "ar - shortcut to call Rbitter::Record"
19
+ puts "connect_ar - Prepare Rbitter::Record to be ready"
20
+ puts "csv_backup - export Rbitter::Record into comma-separated values"
21
+ puts "help - to show this message again"
22
+ puts "xmlrpc - send xmlrpc command to destination"
23
+ puts "xmlrpc_dest - set destination for xmlrpc command"
24
+ puts "^D, 'exit' to exit from here"
25
+ end
26
+
27
+ def connect_ar
28
+ ARSupport.connect_database
29
+ puts "Rbitter::Record is ready."
30
+ end
31
+
32
+ def csv_backup *args
33
+ if args.length < 0
34
+ puts "Usage: csv_backup('filename.csv')"
35
+ puts "Estimated running time depends on system environment"
36
+ else
37
+ ARSupport.export_to_csv(args[0])
38
+ end
39
+ end
40
+
41
+ def ar
42
+ Rbitter::Record
43
+ end
44
+
45
+ def exit
46
+ Kernel.exit(0)
47
+ end
48
+
49
+ def xmlrpc_dest args={}
50
+ if args.empty?
51
+ puts "Usage: xmlrpc_dest({ :rpchost => '', :rpcpath => '', :rpcport => 1400,"
52
+ puts " :xmlrpc_auth_id => '', xmlrpc_auth_password => '' })"
53
+ end
54
+
55
+ @rpchost = args.fetch(:rpchost) { "127.0.0.1" }
56
+ @rpcpath = args.fetch(:rpcpath) { "/" }
57
+ @rpcport = args.fetch(:rpcport) { 1400 }
58
+
59
+ cl = XMLRPC::Client.new(@rpchost, @rpcpath, @rpcport)
60
+ @xmlrpc_cookie = "auth_key=" + cl.call('rbitter.auth',
61
+ args.fetch(:xmlrpc_auth_id) { Rbitter.env['xmlrpc']['auth'][0] },
62
+ args.fetch(:xmlrpc_auth_password) { Rbitter.env['xmlrpc']['auth'][1] } )
63
+
64
+ if @xmlrpc_cookie != "auth_key="
65
+ puts "Authentication completed"
66
+ else
67
+ puts "Authentication failed"
68
+ end
69
+ end
70
+
71
+ def xmlrpc *args
72
+ if args.empty?
73
+ puts "Usage: xmlrpc(command, [params in Array])"
74
+ puts "Ex) xmlrpc(\'rbitter.echo\',' [\"Hello World!\"])"
75
+ puts "Please configure XMLRPC destination with xmlrpc_dest method"
76
+ return false
77
+ end
78
+
79
+ cl = XMLRPC::Client.new(@rpchost, @rpcpath, @rpcport, @xmlrpc_cookie)
80
+
81
+ if args.length <= 1 or args[1].nil?
82
+ cl.call(args[0])
83
+ else
84
+ cl.call(args[0], *args[1])
85
+ end
86
+ end
87
+
88
+ def start
89
+ Ripl.start :binding => binding
90
+ end
91
+ end
92
+ end
93
+
@@ -1,41 +1,41 @@
1
- module Rbitter
2
- DEFAULT_CONFIG_JSON = <<-ENDOFJSON
3
- {
4
- "twitter": {
5
- "consumer_key": "",
6
- "consumer_secret": "",
7
- "access_token": "",
8
- "access_token_secret": "",
9
- "connection": {
10
- "reconnect": true,
11
- "timeout_secs": 5
12
- }
13
- },
14
- "activerecord": "sqlite3",
15
- "sqlite3": {
16
- "dbfile": "rbitter.sqlite"
17
- },
18
- "mysql2": {
19
- "host": "localhost",
20
- "port": 3306,
21
- "dbname": "archive",
22
- "username": "",
23
- "password": ""
24
- },
25
- "media_downloader": {
26
- "large_image": true,
27
- "download_dir": "imgs/"
28
- },
29
- "xmlrpc": {
30
- "enable": true,
31
- "bind_host": "0.0.0.0",
32
- "bind_port": 1400,
33
- "auth": {
34
- "username": "username",
35
- "password": "password"
36
- },
37
- "handles": ["/path/to/handles"]
38
- }
39
- }
40
- ENDOFJSON
41
- end
1
+ module Rbitter
2
+ DEFAULT_CONFIG_JSON = <<-ENDOFJSON
3
+ {
4
+ "twitter": {
5
+ "consumer_key": "",
6
+ "consumer_secret": "",
7
+ "access_token": "",
8
+ "access_token_secret": "",
9
+ "connection": {
10
+ "reconnect": true,
11
+ "timeout_secs": 5
12
+ }
13
+ },
14
+ "activerecord": "sqlite3",
15
+ "sqlite3": {
16
+ "dbfile": "rbitter.sqlite"
17
+ },
18
+ "mysql2": {
19
+ "host": "localhost",
20
+ "port": 3306,
21
+ "dbname": "archive",
22
+ "username": "",
23
+ "password": ""
24
+ },
25
+ "media_downloader": {
26
+ "large_image": true,
27
+ "download_dir": "imgs/"
28
+ },
29
+ "xmlrpc": {
30
+ "enable": true,
31
+ "bind_host": "0.0.0.0",
32
+ "bind_port": 1400,
33
+ "auth": {
34
+ "username": "username",
35
+ "password": "password"
36
+ },
37
+ "handles": ["/path/to/handles"]
38
+ }
39
+ }
40
+ ENDOFJSON
41
+ end
@@ -1,63 +1,63 @@
1
- # encoding: utf-8
2
-
3
- require "net/http"
4
- require "openssl"
5
-
6
- module Rbitter
7
- class DLThread
8
- def initialize(dlfolder, large_flag)
9
- @dest = dlfolder
10
- if not File.directory?(dlfolder)
11
- warn "[dlthread] Given download location is not available for downloading."
12
- warn "[dlthread] Fallback to current directory."
13
- @dest = "./"
14
- end
15
-
16
- if large_flag.nil?
17
- @large_image = false
18
- else
19
- @large_image = large_flag
20
- end
21
-
22
- @pool = Array.new
23
- end
24
-
25
- def <<(url_array)
26
- download_task = Thread.new {
27
- url_array.each { |url|
28
- uri = URI.parse(@large_image ? url + ":large" : url)
29
- ssl = uri.scheme.downcase == 'https'
30
-
31
- Net::HTTP.start(uri.host, uri.port, :use_ssl => ssl) { |h|
32
- req = Net::HTTP::Get.new uri.request_uri
33
- h.request(req) { |res|
34
- case res
35
- when Net::HTTPOK
36
- fname = File.basename(url)
37
-
38
- puts "[fetch] remote: #{uri.path} => local: #{fname}"
39
- open(File.join(@dest, fname), "wb") { |file|
40
- res.read_body { |chunk| file.write(chunk) }
41
- }
42
- end
43
- }
44
- }
45
- }
46
- }
47
-
48
- @pool.push download_task
49
- end
50
-
51
- def job_cleanup
52
- until @pool.empty?
53
- dlthrd = @pool.shift
54
-
55
- if dlthrd.alive?
56
- puts "[dlthread] Thread forceful cleaning up [remains: #{@pool.length}]"
57
- dlthrd.terminate
58
- dlthrd.join
59
- end
60
- end
61
- end
62
- end
1
+ # encoding: utf-8
2
+
3
+ require "net/http"
4
+ require "openssl"
5
+
6
+ module Rbitter
7
+ class DLThread
8
+ def initialize(dlfolder, large_flag)
9
+ @dest = dlfolder
10
+ if not File.directory?(dlfolder)
11
+ warn "[dlthread] Given download location is not available for downloading."
12
+ warn "[dlthread] Fallback to current directory."
13
+ @dest = "./"
14
+ end
15
+
16
+ if large_flag.nil?
17
+ @large_image = false
18
+ else
19
+ @large_image = large_flag
20
+ end
21
+
22
+ @pool = Array.new
23
+ end
24
+
25
+ def <<(url_array)
26
+ download_task = Thread.new {
27
+ url_array.each { |url|
28
+ uri = URI.parse(@large_image ? url + ":large" : url)
29
+ ssl = uri.scheme.downcase == 'https'
30
+
31
+ Net::HTTP.start(uri.host, uri.port, :use_ssl => ssl) { |h|
32
+ req = Net::HTTP::Get.new uri.request_uri
33
+ h.request(req) { |res|
34
+ case res
35
+ when Net::HTTPOK
36
+ fname = File.basename(url)
37
+
38
+ puts "[fetch] remote: #{uri.path} => local: #{fname}"
39
+ open(File.join(@dest, fname), "wb") { |file|
40
+ res.read_body { |chunk| file.write(chunk) }
41
+ }
42
+ end
43
+ }
44
+ }
45
+ }
46
+ }
47
+
48
+ @pool.push download_task
49
+ end
50
+
51
+ def job_cleanup
52
+ until @pool.empty?
53
+ dlthrd = @pool.shift
54
+
55
+ if dlthrd.alive?
56
+ puts "[dlthread] Thread forceful cleaning up [remains: #{@pool.length}]"
57
+ dlthrd.terminate
58
+ dlthrd.join
59
+ end
60
+ end
61
+ end
62
+ end
63
63
  end
data/lib/rbitter/env.rb CHANGED
@@ -1,116 +1,116 @@
1
- # encoding: utf-8
2
-
3
- require "json"
4
- require "rbitter/default/config_json"
5
-
6
- module Rbitter
7
- @@env = Hash.new
8
-
9
- class ConfigFileError < StandardError; end
10
- class MissingFieldError < StandardError; end
11
-
12
- def self.[](k)
13
- @@env[k]
14
- end
15
-
16
- module_function
17
- def env
18
- @@env
19
- end
20
-
21
- def env_reset
22
- @@env.clear
23
- end
24
-
25
- def env_listfields hash
26
- path_stack = ['']
27
- generated = []
28
-
29
- until path_stack.empty?
30
- path = path_stack.pop
31
-
32
- if path == ''
33
- o = hash
34
- else
35
- nodes = path.strip.split('->')
36
- o = hash
37
- until nodes.empty?
38
- o = o[nodes.shift]
39
- end
40
- end
41
-
42
- o.each_key { |k|
43
- if o[k].is_a?(Hash)
44
- path_stack << "#{k}" if path.empty?
45
- path_stack << path + "->#{k}" unless path.empty?
46
- else
47
- generated << "#{k}" if path.empty?
48
- generated << path + "->#{k}" unless path.empty?
49
- end
50
- }
51
- end
52
-
53
- generated
54
- end
55
-
56
- def env_valid?
57
- defaults = env_listfields(JSON.parse(DEFAULT_CONFIG_JSON))
58
- currents = env_listfields(@@env)
59
- not_errored = true
60
-
61
- # Cross checking (2 phases)
62
- # In current exists, default does not: redundant configuration
63
- # Level: warning since it is not utilized at all.
64
- currents.each { |conf|
65
- unless defaults.include?(conf)
66
- warn "[config.json] Unused config: #{conf}. You can safely remove it."
67
- end
68
- }
69
-
70
- # In default exists, current does not: missing configuration
71
- # Level: error and program should stop. (return false for this)
72
- defaults.each { |conf|
73
- unless currents.include?(conf)
74
- warn "[config.json] Config not found: #{conf}. Invalid configuration!"
75
- not_errored = false
76
- end
77
- }
78
- not_errored
79
- end
80
-
81
- def config_initialize json_path=nil
82
- env_reset
83
-
84
- unless json_path.nil?
85
- begin
86
- open(json_path, 'r') { |file|
87
- @@env = JSON.parse(file.read)
88
- }
89
-
90
- return @@env if env_valid?
91
- fail StandardError, "Invalid configuration"
92
- rescue => e
93
- fail ConfigFileError, "Load Failure (#{json_path}): #{e.to_s}"
94
- end
95
- end
96
-
97
- # Configuration default location
98
- # 1. (current_dir)/config.json
99
- # 2. (current_dir)/.rbitter/config.json
100
- locations = ["config.json", ".rbitter/config.json"]
101
- locations.collect! { |base| File.join(Dir.pwd, base) }
102
-
103
- for location in locations
104
- next unless File.file?(location)
105
- open(location, 'r') { |file|
106
- @@env = JSON.parse(file.read)
107
- }
108
- break unless @@env.empty?
109
- end
110
-
111
- fail ConfigFileError, "No config.json on #{locations.join(' or ')}" if @@env.empty?
112
- fail ConfigFileError, "Configuration outdated. Please see above messages to update it" if not env_valid?
113
-
114
- puts "[config.json] Loaded configuration is valid. good to go!"
115
- end
116
- end
1
+ # encoding: utf-8
2
+
3
+ require "json"
4
+ require "rbitter/default/config_json"
5
+
6
+ module Rbitter
7
+ @@env = Hash.new
8
+
9
+ class ConfigFileError < StandardError; end
10
+ class MissingFieldError < StandardError; end
11
+
12
+ def self.[](k)
13
+ @@env[k]
14
+ end
15
+
16
+ module_function
17
+ def env
18
+ @@env
19
+ end
20
+
21
+ def env_reset
22
+ @@env.clear
23
+ end
24
+
25
+ def env_listfields hash
26
+ path_stack = ['']
27
+ generated = []
28
+
29
+ until path_stack.empty?
30
+ path = path_stack.pop
31
+
32
+ if path == ''
33
+ o = hash
34
+ else
35
+ nodes = path.strip.split('->')
36
+ o = hash
37
+ until nodes.empty?
38
+ o = o[nodes.shift]
39
+ end
40
+ end
41
+
42
+ o.each_key { |k|
43
+ if o[k].is_a?(Hash)
44
+ path_stack << "#{k}" if path.empty?
45
+ path_stack << path + "->#{k}" unless path.empty?
46
+ else
47
+ generated << "#{k}" if path.empty?
48
+ generated << path + "->#{k}" unless path.empty?
49
+ end
50
+ }
51
+ end
52
+
53
+ generated
54
+ end
55
+
56
+ def env_valid?
57
+ defaults = env_listfields(JSON.parse(DEFAULT_CONFIG_JSON))
58
+ currents = env_listfields(@@env)
59
+ not_errored = true
60
+
61
+ # Cross checking (2 phases)
62
+ # In current exists, default does not: redundant configuration
63
+ # Level: warning since it is not utilized at all.
64
+ currents.each { |conf|
65
+ unless defaults.include?(conf)
66
+ warn "[config.json] Unused config: #{conf}. You can safely remove it."
67
+ end
68
+ }
69
+
70
+ # In default exists, current does not: missing configuration
71
+ # Level: error and program should stop. (return false for this)
72
+ defaults.each { |conf|
73
+ unless currents.include?(conf)
74
+ warn "[config.json] Config not found: #{conf}. Invalid configuration!"
75
+ not_errored = false
76
+ end
77
+ }
78
+ not_errored
79
+ end
80
+
81
+ def config_initialize json_path=nil
82
+ env_reset
83
+
84
+ unless json_path.nil?
85
+ begin
86
+ open(json_path, 'r') { |file|
87
+ @@env = JSON.parse(file.read)
88
+ }
89
+
90
+ return @@env if env_valid?
91
+ fail StandardError, "Invalid configuration"
92
+ rescue => e
93
+ fail ConfigFileError, "Load Failure (#{json_path}): #{e.to_s}"
94
+ end
95
+ end
96
+
97
+ # Configuration default location
98
+ # 1. (current_dir)/config.json
99
+ # 2. (current_dir)/.rbitter/config.json
100
+ locations = ["config.json", ".rbitter/config.json"]
101
+ locations.collect! { |base| File.join(Dir.pwd, base) }
102
+
103
+ for location in locations
104
+ next unless File.file?(location)
105
+ open(location, 'r') { |file|
106
+ @@env = JSON.parse(file.read)
107
+ }
108
+ break unless @@env.empty?
109
+ end
110
+
111
+ fail ConfigFileError, "No config.json on #{locations.join(' or ')}" if @@env.empty?
112
+ fail ConfigFileError, "Configuration outdated. Please see above messages to update it" if not env_valid?
113
+
114
+ puts "[config.json] Loaded configuration is valid. good to go!"
115
+ end
116
+ end
@@ -1,8 +1,8 @@
1
- # encoding: utf-8
2
- require 'socket'
3
-
4
- class Socket
5
- def self.ip_address_list
6
- fail NotImplementedError
7
- end
8
- end
1
+ # encoding: utf-8
2
+ require 'socket'
3
+
4
+ class Socket
5
+ def self.ip_address_list
6
+ fail NotImplementedError
7
+ end
8
+ end