rbitter 0.2.0-java → 0.2.1-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +23 -22
  3. data/.rspec +2 -2
  4. data/.travis.yml +15 -15
  5. data/Gemfile +12 -12
  6. data/LICENSE.txt +22 -22
  7. data/Rakefile +8 -8
  8. data/bin/rbitter +20 -20
  9. data/lib/rbitter/arcserver.rb +169 -165
  10. data/lib/rbitter/console.rb +93 -93
  11. data/lib/rbitter/default/config_json.rb +41 -41
  12. data/lib/rbitter/dlthread.rb +62 -62
  13. data/lib/rbitter/env.rb +116 -116
  14. data/lib/rbitter/override/gems/rubysl-socket/socket.rb +8 -8
  15. data/lib/rbitter/override/gems/twitter/connection.rb +45 -45
  16. data/lib/rbitter/override.rb +47 -47
  17. data/lib/rbitter/progress.rb +23 -0
  18. data/lib/rbitter/records.rb +127 -127
  19. data/lib/rbitter/records_migrate/20150327_add_index.rb +11 -11
  20. data/lib/rbitter/records_migrate/20150504_add_replyto_column.rb +11 -11
  21. data/lib/rbitter/streaming.rb +104 -104
  22. data/lib/rbitter/version.rb +20 -20
  23. data/lib/rbitter/xmlrpc.rb +3 -3
  24. data/lib/rbitter/xmlrpcd/base.rb +24 -24
  25. data/lib/rbitter/xmlrpcd/rpchandles.rb +11 -11
  26. data/lib/rbitter/xmlrpcd/xmlrpc_auth_server.rb +82 -82
  27. data/lib/rbitter/xmlrpcd/xmlrpcd.rb +69 -69
  28. data/lib/rbitter.rb +62 -62
  29. data/rbitter.gemspec +46 -46
  30. data/spec/config/default.json +32 -32
  31. data/spec/rbitter/arcserver_spec.rb +30 -30
  32. data/spec/rbitter/console_spec.rb +9 -9
  33. data/spec/rbitter/default/config_json_spec.rb +3 -3
  34. data/spec/rbitter/dlthread_spec.rb +8 -8
  35. data/spec/rbitter/env_spec.rb +76 -76
  36. data/spec/rbitter/override/gems/twitter/connection_spec.rb +8 -8
  37. data/spec/rbitter/progress_spec.rb +1 -0
  38. data/spec/rbitter/records_spec.rb +13 -13
  39. data/spec/rbitter/streaming_spec.rb +9 -9
  40. data/spec/rbitter/version_spec.rb +8 -8
  41. data/spec/rbitter/xmlrpc_spec.rb +8 -8
  42. data/spec/rbitter/xmlrpcd/base_spec.rb +29 -29
  43. data/spec/rbitter/xmlrpcd/rpchandles_spec.rb +10 -10
  44. data/spec/rbitter/xmlrpcd/xmlrpc_auth_server_spec.rb +8 -8
  45. data/spec/rbitter/xmlrpcd/xmlrpcd_spec.rb +9 -9
  46. data/spec/rbitter_spec.rb +38 -38
  47. data/spec/spec_helper.rb +39 -39
  48. metadata +6 -3
@@ -1,105 +1,105 @@
1
- # encoding: utf-8
2
-
3
- require 'twitter'
4
-
5
- module Rbitter
6
- class DummyStreamClient
7
- def initialize(tokens); end
8
-
9
- def run(&operation_block)
10
- internal(&operation_block)
11
- end
12
-
13
- private
14
- def internal(&operation_block)
15
- tweets = [{
16
- "tweetid" => 1,
17
- "userid" => 1,
18
- "replyto" => nil,
19
- "tweet" => "test",
20
- "rt_count" => 0,
21
- "fav_count" => 0,
22
- "screen_name" => "twitter",
23
- "date" => "2015-01-01 12:11:10",
24
- "media_urls" => ["https://pbs.twimg.com/media/CEPWFtgUgAEmbcV.png"],
25
- "web_urls" => ["https://www.google.com/"]
26
- }]
27
-
28
- tweets.each { |tweet|
29
- yield tweet
30
- }
31
- end
32
- end
33
-
34
- class StreamClient
35
- def initialize(tokens)
36
- @t = Twitter::Streaming::Client.new do |object|
37
- object.consumer_key = tokens['consumer_key']
38
- object.consumer_secret = tokens['consumer_secret']
39
- object.access_token = tokens['access_token']
40
- object.access_token_secret = tokens['access_token_secret']
41
- end
42
- end
43
-
44
- def run(&operation_block)
45
- begin
46
- internal(&operation_block)
47
- rescue EOFError => e
48
- puts "Network unreachable. Retry in 3 seconds..."
49
- sleep 3
50
- retry
51
- end
52
- end
53
-
54
- private
55
- def internal(&operation_block)
56
- @t.user do |tweet|
57
- if tweet.is_a?(Twitter::Tweet)
58
- if tweet.retweet?
59
- tweet = tweet.retweeted_tweet
60
- end
61
-
62
- text = tweet.full_text.gsub(/(\r\n|\n)/, '')
63
-
64
- # unpack uris and media links
65
- media_urls = Array.new
66
- web_urls = Array.new
67
-
68
- if tweet.entities?
69
- if tweet.media?
70
- tweet.media.each { |media|
71
- media_urls.push("#{media.media_uri_https}")
72
- text.gsub!("#{media.url}", "#{media.display_url}")
73
- }
74
- end
75
-
76
- text += " "
77
- text += media_urls.join(" ")
78
-
79
- if tweet.uris?
80
- tweet.uris.each { |uri|
81
- web_urls.push("#{uri.expanded_url}")
82
- text.gsub!("#{uri.url}", "#{uri.expanded_url}")
83
- }
84
- end
85
- end
86
-
87
- res = {
88
- "tweetid" => tweet.id,
89
- "userid" => tweet.user.id,
90
- "replyto" => tweet.in_reply_to_status_id? ? tweet.in_reply_to_status_id : nil,
91
- "tweet" => text,
92
- "rt_count" => tweet.retweet_count,
93
- "fav_count" => tweet.favorite_count,
94
- "screen_name" => tweet.user.screen_name,
95
- "date" => tweet.created_at,
96
- "media_urls" => media_urls,
97
- "web_urls" => web_urls
98
- }
99
-
100
- yield res
101
- end
102
- end
103
- end
104
- end
1
+ # encoding: utf-8
2
+
3
+ require 'twitter'
4
+
5
+ module Rbitter
6
+ class DummyStreamClient
7
+ def initialize(tokens); end
8
+
9
+ def run(&operation_block)
10
+ internal(&operation_block)
11
+ end
12
+
13
+ private
14
+ def internal(&operation_block)
15
+ tweets = [{
16
+ "tweetid" => 1,
17
+ "userid" => 1,
18
+ "replyto" => nil,
19
+ "tweet" => "test",
20
+ "rt_count" => 0,
21
+ "fav_count" => 0,
22
+ "screen_name" => "twitter",
23
+ "date" => "2015-01-01 12:11:10",
24
+ "media_urls" => ["https://pbs.twimg.com/media/CEPWFtgUgAEmbcV.png"],
25
+ "web_urls" => ["https://www.google.com/"]
26
+ }]
27
+
28
+ tweets.each { |tweet|
29
+ yield tweet
30
+ }
31
+ end
32
+ end
33
+
34
+ class StreamClient
35
+ def initialize(tokens)
36
+ @t = Twitter::Streaming::Client.new do |object|
37
+ object.consumer_key = tokens['consumer_key']
38
+ object.consumer_secret = tokens['consumer_secret']
39
+ object.access_token = tokens['access_token']
40
+ object.access_token_secret = tokens['access_token_secret']
41
+ end
42
+ end
43
+
44
+ def run(&operation_block)
45
+ begin
46
+ internal(&operation_block)
47
+ rescue EOFError => e
48
+ puts "Network unreachable. Retry in 3 seconds..."
49
+ sleep 3
50
+ retry
51
+ end
52
+ end
53
+
54
+ private
55
+ def internal(&operation_block)
56
+ @t.user do |tweet|
57
+ if tweet.is_a?(Twitter::Tweet)
58
+ if tweet.retweet?
59
+ tweet = tweet.retweeted_tweet
60
+ end
61
+
62
+ text = tweet.full_text.gsub(/(\r\n|\n)/, '')
63
+
64
+ # unpack uris and media links
65
+ media_urls = Array.new
66
+ web_urls = Array.new
67
+
68
+ if tweet.entities?
69
+ if tweet.media?
70
+ tweet.media.each { |media|
71
+ media_urls.push("#{media.media_uri_https}")
72
+ text.gsub!("#{media.url}", "#{media.display_url}")
73
+ }
74
+ end
75
+
76
+ text += " "
77
+ text += media_urls.join(" ")
78
+
79
+ if tweet.uris?
80
+ tweet.uris.each { |uri|
81
+ web_urls.push("#{uri.expanded_url}")
82
+ text.gsub!("#{uri.url}", "#{uri.expanded_url}")
83
+ }
84
+ end
85
+ end
86
+
87
+ res = {
88
+ "tweetid" => tweet.id,
89
+ "userid" => tweet.user.id,
90
+ "replyto" => tweet.in_reply_to_status_id? ? tweet.in_reply_to_status_id : nil,
91
+ "tweet" => text,
92
+ "rt_count" => tweet.retweet_count,
93
+ "fav_count" => tweet.favorite_count,
94
+ "screen_name" => tweet.user.screen_name,
95
+ "date" => tweet.created_at,
96
+ "media_urls" => media_urls,
97
+ "web_urls" => web_urls
98
+ }
99
+
100
+ yield res
101
+ end
102
+ end
103
+ end
104
+ end
105
105
  end
@@ -1,20 +1,20 @@
1
- module Rbitter
2
- PRODUCT_NAME = "Rbitter"
3
- VERSION = "0.2.0"
4
-
5
- def major
6
- VERSION.match(/^([0-9]+)\./)[1]
7
- end
8
-
9
- def minor
10
- VERSION.match(/\.([0-9]+)\./)[1]
11
- end
12
-
13
- def patchlv
14
- VERSION.match(/\.([0-9]+)$/)[1]
15
- end
16
-
17
- def version_string
18
- "#{PRODUCT_NAME} #{VERSION}"
19
- end
20
- end
1
+ module Rbitter
2
+ PRODUCT_NAME = "Rbitter"
3
+ VERSION = "0.2.1"
4
+
5
+ def major
6
+ VERSION.match(/^([0-9]+)\./)[1]
7
+ end
8
+
9
+ def minor
10
+ VERSION.match(/\.([0-9]+)\./)[1]
11
+ end
12
+
13
+ def patchlv
14
+ VERSION.match(/\.([0-9]+)$/)[1]
15
+ end
16
+
17
+ def version_string
18
+ "#{PRODUCT_NAME} #{VERSION}"
19
+ end
20
+ end
@@ -1,4 +1,4 @@
1
- # encoding: utf-8
2
- #
3
-
1
+ # encoding: utf-8
2
+ #
3
+
4
4
  require "rbitter/xmlrpcd/xmlrpcd"
@@ -1,25 +1,25 @@
1
- # encoding: utf-8
2
-
3
- module RPCHandles
4
- RH_INFO = Struct.new("RPCHANDLE_INFO", :name, :version, :author, :description) {
5
- def digest
6
- "<rpchandle: #{name}-#{version} (written by #{author}, #{description})>"
7
- end
8
- }
9
-
10
- module BaseHandle
11
- # If a handler doesn't require an authorization, please inherit below class
12
- class NoAuth < Object
13
- def self.auth?
14
- false
15
- end
16
- end
17
-
18
- # If a handler does require an authorization, please inherit below class
19
- class Auth < Object
20
- def self.auth?
21
- true
22
- end
23
- end
24
- end
1
+ # encoding: utf-8
2
+
3
+ module RPCHandles
4
+ RH_INFO = Struct.new("RPCHANDLE_INFO", :name, :version, :author, :description) {
5
+ def digest
6
+ "<rpchandle: #{name}-#{version} (written by #{author}, #{description})>"
7
+ end
8
+ }
9
+
10
+ module BaseHandle
11
+ # If a handler doesn't require an authorization, please inherit below class
12
+ class NoAuth < Object
13
+ def self.auth?
14
+ false
15
+ end
16
+ end
17
+
18
+ # If a handler does require an authorization, please inherit below class
19
+ class Auth < Object
20
+ def self.auth?
21
+ true
22
+ end
23
+ end
24
+ end
25
25
  end
@@ -1,12 +1,12 @@
1
- # encoding: utf-8
2
-
3
- module RPCHandles
4
- # Override this function will activate authentication feature.
5
- # You can write and add RPCHandle. See 'rpc' folder.
6
-
7
- @@auth_pool = nil
8
- module_function
9
- def auth
10
- @@auth_pool
11
- end
1
+ # encoding: utf-8
2
+
3
+ module RPCHandles
4
+ # Override this function will activate authentication feature.
5
+ # You can write and add RPCHandle. See 'rpc' folder.
6
+
7
+ @@auth_pool = nil
8
+ module_function
9
+ def auth
10
+ @@auth_pool
11
+ end
12
12
  end
@@ -1,83 +1,83 @@
1
- # encoding: utf-8
2
-
3
- require "rbitter/xmlrpcd/rpchandles"
4
- require "rbitter/xmlrpcd/base"
5
- require "xmlrpc/server"
6
- require "webrick"
7
-
8
- module XMLRPC
9
- class HTTPAuthXMLRPCServer < XMLRPC::WEBrickServlet
10
- def extract_method(methodname, *args)
11
- for name, obj in @handler
12
- if obj.kind_of? Proc
13
- next unless methodname == name
14
- else
15
- next unless methodname =~ /^#{name}(.+)$/
16
- next unless obj.respond_to? $1
17
- return obj.method($1)
18
- end
19
- end
20
- nil
21
- end
22
-
23
- def service(request, response)
24
- # Taken from xmlrpc/server.rb
25
- if @valid_ip
26
- raise WEBrick::HTTPStatus::Forbidden unless @valid_ip.any? { |ip| request.peeraddr[3] =~ ip }
27
- end
28
-
29
- if request.request_method != "POST"
30
- raise WEBrick::HTTPStatus::MethodNotAllowed,
31
- "unsupported method `#{request.request_method}'."
32
- end
33
-
34
- if parse_content_type(request['Content-type']).first != "text/xml"
35
- raise WEBrick::HTTPStatus::BadRequest
36
- end
37
-
38
- length = (request['Content-length'] || 0).to_i
39
-
40
- raise WEBrick::HTTPStatus::LengthRequired unless length > 0
41
-
42
- data = request.body
43
-
44
- if data.nil? or data.bytesize != length
45
- raise WEBrick::HTTPStatus::BadRequest
46
- end
47
-
48
- # Originally, process(data) was here.
49
- # We need to check whether a method requires authorization.
50
- rpc_method_name, rpc_params = parser().parseMethodCall(data)
51
- rpc_method = extract_method(rpc_method_name)
52
-
53
- if RPCHandles.auth.nil?
54
- resp = handle(rpc_method_name, *rpc_params)
55
- else
56
- if rpc_method.owner.ancestors.include?(RPCHandles::BaseHandle::Auth)
57
- # Check cookie and check it's valid
58
- if request.cookies.size == 1 \
59
- and request.cookies[0].name == "auth_key" \
60
- and RPCHandles.auth.include?(request.cookies[0].value)
61
- resp = handle(rpc_method_name, *rpc_params)
62
- else
63
- # Permission required
64
- raise WEBrick::HTTPStatus::Forbidden
65
- end
66
- elsif rpc_method.owner.ancestors.include?(RPCHandles::BaseHandle::NoAuth)
67
- resp = handle(rpc_method_name, *rpc_params)
68
- else
69
- raise WEBrick::HTTPStatus::Forbidden
70
- end
71
- end
72
-
73
- if resp.nil? or resp.bytesize <= 0
74
- raise WEBrick::HTTPStatus::InternalServerError
75
- end
76
-
77
- response.status = 200
78
- response['Content-Length'] = resp.bytesize
79
- response['Content-Type'] = "text/xml; charset=utf-8"
80
- response.body = resp
81
- end
82
- end
1
+ # encoding: utf-8
2
+
3
+ require "rbitter/xmlrpcd/rpchandles"
4
+ require "rbitter/xmlrpcd/base"
5
+ require "xmlrpc/server"
6
+ require "webrick"
7
+
8
+ module XMLRPC
9
+ class HTTPAuthXMLRPCServer < XMLRPC::WEBrickServlet
10
+ def extract_method(methodname, *args)
11
+ for name, obj in @handler
12
+ if obj.kind_of? Proc
13
+ next unless methodname == name
14
+ else
15
+ next unless methodname =~ /^#{name}(.+)$/
16
+ next unless obj.respond_to? $1
17
+ return obj.method($1)
18
+ end
19
+ end
20
+ nil
21
+ end
22
+
23
+ def service(request, response)
24
+ # Taken from xmlrpc/server.rb
25
+ if @valid_ip
26
+ raise WEBrick::HTTPStatus::Forbidden unless @valid_ip.any? { |ip| request.peeraddr[3] =~ ip }
27
+ end
28
+
29
+ if request.request_method != "POST"
30
+ raise WEBrick::HTTPStatus::MethodNotAllowed,
31
+ "unsupported method `#{request.request_method}'."
32
+ end
33
+
34
+ if parse_content_type(request['Content-type']).first != "text/xml"
35
+ raise WEBrick::HTTPStatus::BadRequest
36
+ end
37
+
38
+ length = (request['Content-length'] || 0).to_i
39
+
40
+ raise WEBrick::HTTPStatus::LengthRequired unless length > 0
41
+
42
+ data = request.body
43
+
44
+ if data.nil? or data.bytesize != length
45
+ raise WEBrick::HTTPStatus::BadRequest
46
+ end
47
+
48
+ # Originally, process(data) was here.
49
+ # We need to check whether a method requires authorization.
50
+ rpc_method_name, rpc_params = parser().parseMethodCall(data)
51
+ rpc_method = extract_method(rpc_method_name)
52
+
53
+ if RPCHandles.auth.nil?
54
+ resp = handle(rpc_method_name, *rpc_params)
55
+ else
56
+ if rpc_method.owner.ancestors.include?(RPCHandles::BaseHandle::Auth)
57
+ # Check cookie and check it's valid
58
+ if request.cookies.size == 1 \
59
+ and request.cookies[0].name == "auth_key" \
60
+ and RPCHandles.auth.include?(request.cookies[0].value)
61
+ resp = handle(rpc_method_name, *rpc_params)
62
+ else
63
+ # Permission required
64
+ raise WEBrick::HTTPStatus::Forbidden
65
+ end
66
+ elsif rpc_method.owner.ancestors.include?(RPCHandles::BaseHandle::NoAuth)
67
+ resp = handle(rpc_method_name, *rpc_params)
68
+ else
69
+ raise WEBrick::HTTPStatus::Forbidden
70
+ end
71
+ end
72
+
73
+ if resp.nil? or resp.bytesize <= 0
74
+ raise WEBrick::HTTPStatus::InternalServerError
75
+ end
76
+
77
+ response.status = 200
78
+ response['Content-Length'] = resp.bytesize
79
+ response['Content-Type'] = "text/xml; charset=utf-8"
80
+ response.body = resp
81
+ end
82
+ end
83
83
  end
@@ -1,69 +1,69 @@
1
- # encoding: utf-8
2
-
3
- require "rbitter/xmlrpcd/rpchandles"
4
- require "rbitter/xmlrpcd/xmlrpc_auth_server"
5
- require "webrick"
6
-
7
- module Rbitter
8
- RPC_PREFIX="rbitter"
9
-
10
- class RPCServer
11
- def initialize bind_host, bind_port
12
- @server = WEBrick::HTTPServer.new(:Port => bind_port.to_i, :BindAddress => bind_host.to_s, :MaxClients => 4, :Logger => WEBrick::Log.new($stdout))
13
- @core = XMLRPC::HTTPAuthXMLRPCServer.new
14
- @core.set_default_handler { |name, *args|
15
- "NO_COMMAND: #{name} with args #{args.inspect}"
16
- }
17
- end
18
-
19
- def load_all_handles
20
- Rbitter["xmlrpc"]["handles"].each { |path|
21
- puts "[xmlrpc] Scanning handles from (#{path})"
22
- Dir.entries(path).each { |fname|
23
- fname = File.join(path, fname)
24
- if File.exist?(fname) and File.file?(fname)
25
- if fname.match(/rh_\w+\.rb$/)
26
- begin
27
- load fname
28
- rescue Exception => e
29
- # stub
30
- puts "Exception while loading #{fname}"
31
- puts e.inspect
32
- end
33
- else
34
- puts "Ignored: #{fname}"
35
- end
36
- end
37
- }
38
- }
39
-
40
- puts "[xmlrpc] found #{RPCHandles.constants.length} constants."
41
- RPCHandles.constants.each { |handler|
42
- if RPCHandles.const_get(handler).is_a?(Class)
43
- @core.add_handler(RPC_PREFIX, RPCHandles.const_get(handler).new)
44
- end
45
- }
46
- end
47
-
48
- def main_loop
49
- load_all_handles
50
-
51
- @server.mount("/", @core)
52
- @server.start
53
-
54
- puts "[xmlrpc] XMLRPC started"
55
- end
56
- end
57
-
58
- class DummyRPCServer
59
- def initialize bind_host, bind_port; end
60
-
61
- def load_all_handles; end
62
-
63
- def main_loop
64
- puts "[xmlrpc] DummyRPCServer started"
65
- end
66
- end
67
-
68
- class NullRPCServer; end
69
- end
1
+ # encoding: utf-8
2
+
3
+ require "rbitter/xmlrpcd/rpchandles"
4
+ require "rbitter/xmlrpcd/xmlrpc_auth_server"
5
+ require "webrick"
6
+
7
+ module Rbitter
8
+ RPC_PREFIX="rbitter"
9
+
10
+ class RPCServer
11
+ def initialize bind_host, bind_port
12
+ @server = WEBrick::HTTPServer.new(:Port => bind_port.to_i, :BindAddress => bind_host.to_s, :MaxClients => 4, :Logger => WEBrick::Log.new($stdout))
13
+ @core = XMLRPC::HTTPAuthXMLRPCServer.new
14
+ @core.set_default_handler { |name, *args|
15
+ "NO_COMMAND: #{name} with args #{args.inspect}"
16
+ }
17
+ end
18
+
19
+ def load_all_handles
20
+ Rbitter["xmlrpc"]["handles"].each { |path|
21
+ puts "[xmlrpc] Scanning handles from (#{path})"
22
+ Dir.entries(path).each { |fname|
23
+ fname = File.join(path, fname)
24
+ if File.exist?(fname) and File.file?(fname)
25
+ if fname.match(/rh_\w+\.rb$/)
26
+ begin
27
+ load fname
28
+ rescue Exception => e
29
+ # stub
30
+ puts "Exception while loading #{fname}"
31
+ puts e.inspect
32
+ end
33
+ else
34
+ puts "Ignored: #{fname}"
35
+ end
36
+ end
37
+ }
38
+ }
39
+
40
+ puts "[xmlrpc] found #{RPCHandles.constants.length} constants."
41
+ RPCHandles.constants.each { |handler|
42
+ if RPCHandles.const_get(handler).is_a?(Class)
43
+ @core.add_handler(RPC_PREFIX, RPCHandles.const_get(handler).new)
44
+ end
45
+ }
46
+ end
47
+
48
+ def main_loop
49
+ load_all_handles
50
+
51
+ @server.mount("/", @core)
52
+ @server.start
53
+
54
+ puts "[xmlrpc] XMLRPC started"
55
+ end
56
+ end
57
+
58
+ class DummyRPCServer
59
+ def initialize bind_host, bind_port; end
60
+
61
+ def load_all_handles; end
62
+
63
+ def main_loop
64
+ puts "[xmlrpc] DummyRPCServer started"
65
+ end
66
+ end
67
+
68
+ class NullRPCServer; end
69
+ end