rbitter 0.1.0 → 0.1.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +21 -21
  3. data/.rspec +2 -2
  4. data/.travis.yml +15 -15
  5. data/Gemfile +12 -12
  6. data/LICENSE.txt +22 -22
  7. data/Rakefile +8 -8
  8. data/XMLRPC.md +19 -19
  9. data/bin/rbitter +20 -20
  10. data/lib/rbitter/arcserver.rb +130 -126
  11. data/lib/rbitter/console.rb +93 -93
  12. data/lib/rbitter/default/config_json.rb +37 -38
  13. data/lib/rbitter/dlthread.rb +62 -65
  14. data/lib/rbitter/env.rb +62 -62
  15. data/lib/rbitter/libtwitter_connection_override.rb +45 -45
  16. data/lib/rbitter/records.rb +121 -121
  17. data/lib/rbitter/records_migrate/20150327_add_index.rb +11 -11
  18. data/lib/rbitter/records_migrate/20150504_add_replyto_column.rb +11 -11
  19. data/lib/rbitter/streaming.rb +104 -104
  20. data/lib/rbitter/version.rb +20 -20
  21. data/lib/rbitter/xmlrpc.rb +3 -3
  22. data/lib/rbitter/xmlrpcd/base.rb +24 -24
  23. data/lib/rbitter/xmlrpcd/rpchandles.rb +11 -11
  24. data/lib/rbitter/xmlrpcd/xmlrpc_auth_server.rb +82 -82
  25. data/lib/rbitter/xmlrpcd/xmlrpcd.rb +69 -69
  26. data/lib/rbitter.rb +86 -86
  27. data/rbitter.gemspec +46 -42
  28. data/spec/config/default.json +32 -32
  29. data/spec/rbitter/arcserver_spec.rb +30 -30
  30. data/spec/rbitter/console_spec.rb +9 -9
  31. data/spec/rbitter/default/config_json_spec.rb +3 -3
  32. data/spec/rbitter/dlthread_spec.rb +8 -13
  33. data/spec/rbitter/env_spec.rb +62 -62
  34. data/spec/rbitter/libtwitter_connection_override_spec.rb +8 -8
  35. data/spec/rbitter/records_spec.rb +13 -13
  36. data/spec/rbitter/streaming_spec.rb +9 -9
  37. data/spec/rbitter/version_spec.rb +8 -8
  38. data/spec/rbitter/xmlrpc_spec.rb +8 -8
  39. data/spec/rbitter/xmlrpcd/base_spec.rb +29 -29
  40. data/spec/rbitter/xmlrpcd/rpchandles_spec.rb +10 -10
  41. data/spec/rbitter/xmlrpcd/xmlrpc_auth_server_spec.rb +8 -8
  42. data/spec/rbitter/xmlrpcd/xmlrpcd_spec.rb +9 -9
  43. data/spec/rbitter_spec.rb +42 -42
  44. data/spec/spec_helper.rb +39 -39
  45. metadata +29 -56
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 44878b7491619cb57321ab471e8b03279180932b
4
- data.tar.gz: a1dbfc5622f0d65b9aa256992a18b8523d425b0d
3
+ metadata.gz: 3ca2dd0e8a08659416b4e1ba154375673bfe697d
4
+ data.tar.gz: d60daa351fef6fdfa8cc3fdb444b31e10a820c80
5
5
  SHA512:
6
- metadata.gz: d82fc7a0b2158f54aacd644d31aad960614f89615b0c4b471fab30d3fd304627cb2f72aae757d141af4bc4267c6bbf7d0140b68f249833a85bdcebd346feead1
7
- data.tar.gz: a7c43429c676d18247d8fb274956e289ef537b7fe832f8240f869c0748263e36b6e14ab3702bc72e2b81dd5fec9723a83c8de25e50d8ce4214be7bf14027857e
6
+ metadata.gz: 251d4bda5d89583701ca2ecb3b52c5cafeab42b6e3f01d90692e2457d48f53e22ff9cda790daed72b63088cd077c4ba7ba27e8dd0751e9d2b5f1dbbf68dd1286
7
+ data.tar.gz: cb04a2013000e2581b5b8c6dd5b30b15058c6f70f995fbde2dbf31733e78a23c7b8526cafc203cae9d0125f16752c8728cc45b3103f28a83e2d52dbac153118e
data/.gitignore CHANGED
@@ -1,21 +1,21 @@
1
- /.bundle/
2
- /.yardoc
3
- /Gemfile.lock
4
- /_yardoc/
5
- /coverage/
6
- /doc/
7
- /pkg/
8
- /spec/reports/
9
- /tmp/
10
- /build/
11
- /vendor/
12
- *.bundle
13
- *.so
14
- *.o
15
- *.a
16
- mkmf.log
17
- *.gem
18
- *.json
19
- *.sqlite
20
- .rbitter
21
- .rbitter/*
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
10
+ /build/
11
+ /vendor/
12
+ *.bundle
13
+ *.so
14
+ *.o
15
+ *.a
16
+ mkmf.log
17
+ *.gem
18
+ *.json
19
+ *.sqlite
20
+ .rbitter
21
+ .rbitter/*
data/.rspec CHANGED
@@ -1,2 +1,2 @@
1
- --color
2
- --require spec_helper
1
+ --color
2
+ --require spec_helper
data/.travis.yml CHANGED
@@ -1,15 +1,15 @@
1
- language: ruby
2
- rvm:
3
- - 1.9.3
4
- - 2.1.0
5
- - 2.2.0
6
- - jruby-19mode
7
- - jruby-head
8
- - rbx-2
9
-
10
- env: LANG="en_US.UTF-8" LC_ALL="en_US.UTF-8"
11
-
12
- matrix:
13
- allow_failures:
14
- - rvm: ruby-head
15
- - rvm: rbx-2
1
+ language: ruby
2
+ rvm:
3
+ - 1.9.3
4
+ - 2.1.0
5
+ - 2.2.0
6
+ - jruby-19mode
7
+ - jruby-head
8
+ - rbx-2
9
+
10
+ env: LANG="en_US.UTF-8" LC_ALL="en_US.UTF-8"
11
+
12
+ matrix:
13
+ allow_failures:
14
+ - rvm: ruby-head
15
+ - rvm: rbx-2
data/Gemfile CHANGED
@@ -1,12 +1,12 @@
1
- source "https://rubygems.org"
2
-
3
- group :test do
4
- gem 'rspec', '~>3.0'
5
- gem 'simplecov', :require => false, :group => :test
6
- gem 'coveralls', :require => false
7
- end
8
-
9
- gem "bundler"
10
- gem 'jruby-openssl', platforms: :jruby
11
-
12
- gemspec
1
+ source "https://rubygems.org"
2
+
3
+ group :test do
4
+ gem 'rspec', '~>3.0'
5
+ gem 'simplecov', :require => false, :group => :test
6
+ gem 'coveralls', :require => false
7
+ end
8
+
9
+ gem "bundler"
10
+ gem 'jruby-openssl', platforms: :jruby
11
+
12
+ gemspec
data/LICENSE.txt CHANGED
@@ -1,22 +1,22 @@
1
- Copyright (c) 2015 Nidev Plontra
2
-
3
- MIT License
4
-
5
- Permission is hereby granted, free of charge, to any person obtaining
6
- a copy of this software and associated documentation files (the
7
- "Software"), to deal in the Software without restriction, including
8
- without limitation the rights to use, copy, modify, merge, publish,
9
- distribute, sublicense, and/or sell copies of the Software, and to
10
- permit persons to whom the Software is furnished to do so, subject to
11
- the following conditions:
12
-
13
- The above copyright notice and this permission notice shall be
14
- included in all copies or substantial portions of the Software.
15
-
16
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
- LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
- OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
- WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
1
+ Copyright (c) 2015 Nidev Plontra
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/Rakefile CHANGED
@@ -1,8 +1,8 @@
1
- require "bundler/gem_tasks"
2
- require "rspec/core/rake_task"
3
-
4
- RSpec::Core::RakeTask.new
5
-
6
- task :default => :spec
7
- task :basic_test => :spec
8
-
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+
4
+ RSpec::Core::RakeTask.new
5
+
6
+ task :default => :spec
7
+ task :basic_test => :spec
8
+
data/XMLRPC.md CHANGED
@@ -1,19 +1,19 @@
1
- # What is RPC handle ? #
2
- ## Commands ##
3
- ### Authentication ###
4
- ### Revoke Authentication Token ###
5
- ### Echo ###
6
- ### Last Active ###
7
- ### Retriever ###
8
- ### Statistic ###
9
- # How to write own RPC handle? #
10
- RPC handle is a Ruby class. Writing a method in Ruby class, that's it. Names of methods are treated as XMLRPC command.
11
-
12
- When you write a new class for your own RPC handle, you must inherit either Auth or NoAuth class from rpc/base.rb.
13
-
14
- * class Auth < Object: Methods in a Ruby class inheriting Auth requires *auth_key* to access.
15
- * class NoAuth < Object: Methods in a Ruby class inheriting NoAuth doesn't require *auth_key* and these XMLRPC commands can be called by anonymous user.
16
-
17
- Filename should start with 'rh_'. It's prefix to be autoloaded by xmlrpc.rb.
18
-
19
- Refer rpc/rh_echo.rb as an example.
1
+ # What is RPC handle ? #
2
+ ## Commands ##
3
+ ### Authentication ###
4
+ ### Revoke Authentication Token ###
5
+ ### Echo ###
6
+ ### Last Active ###
7
+ ### Retriever ###
8
+ ### Statistic ###
9
+ # How to write own RPC handle? #
10
+ RPC handle is a Ruby class. Writing a method in Ruby class, that's it. Names of methods are treated as XMLRPC command.
11
+
12
+ When you write a new class for your own RPC handle, you must inherit either Auth or NoAuth class from rpc/base.rb.
13
+
14
+ * class Auth < Object: Methods in a Ruby class inheriting Auth requires *auth_key* to access.
15
+ * class NoAuth < Object: Methods in a Ruby class inheriting NoAuth doesn't require *auth_key* and these XMLRPC commands can be called by anonymous user.
16
+
17
+ Filename should start with 'rh_'. It's prefix to be autoloaded by xmlrpc.rb.
18
+
19
+ Refer rpc/rh_echo.rb as an example.
data/bin/rbitter CHANGED
@@ -1,20 +1,20 @@
1
- #!/usr/bin/env ruby
2
-
3
- require "rbitter"
4
-
5
- Rbitter.rbitter_header
6
-
7
- if ARGV.length < 1 || (not Rbitter::BOOTSTRAP_ARGS.include?(ARGV[0]))
8
- Rbitter.rbitter_help_msg
9
- exit -1
10
- else
11
- Rbitter.bootstrap(ARGV)
12
-
13
- if ARGV[0] == "configure"
14
- puts "Writing config.json done"
15
- puts "You can put config.json one of these locations:"
16
- puts "[1] config.json (current folder)"
17
- puts "[2] .rbitter/config.json (current folder)"
18
- end
19
- end
20
-
1
+ #!/usr/bin/env ruby
2
+
3
+ require "rbitter"
4
+
5
+ Rbitter.rbitter_header
6
+
7
+ if ARGV.length < 1 || (not Rbitter::BOOTSTRAP_ARGS.include?(ARGV[0]))
8
+ Rbitter.rbitter_help_msg
9
+ exit -1
10
+ else
11
+ Rbitter.bootstrap(ARGV)
12
+
13
+ if ARGV[0] == "configure"
14
+ puts "Writing config.json done"
15
+ puts "You can put config.json one of these locations:"
16
+ puts "[1] config.json (current folder)"
17
+ puts "[2] .rbitter/config.json (current folder)"
18
+ end
19
+ end
20
+
@@ -1,126 +1,130 @@
1
- # encoding: utf-8
2
-
3
- require "json"
4
- require "date"
5
- require "twitter"
6
- require "resolv"
7
-
8
- require "rbitter/records"
9
- require "rbitter/streaming"
10
- require "rbitter/dlthread"
11
- require "rbitter/xmlrpc"
12
-
13
- module Rbitter
14
- class ArcServer
15
- LOG_NORMAL = 0
16
- LOG_INIT = 1
17
- LOG_HALT = 2
18
-
19
- def initialize(xmlrpcd_class = Rbitter::RPCServer)
20
- @xmlrpcd_class = xmlrpcd_class
21
-
22
- ARSupport.connect_database
23
-
24
- if not ARSupport.prepared?
25
- puts "Initiate database table..."
26
- if Rbitter['activerecord'] == 'mysql2'
27
- ARSupport.prepare "DEFAULT CHARSET=utf8mb4"
28
- else
29
- ARSupport.prepare
30
- end
31
- end
32
-
33
- ARSupport.update_database_scheme
34
-
35
- @dt = DLThread.new(
36
- Rbitter['media_downloader']['download_dir'],
37
- Rbitter['media_downloader']['cacert_path'],
38
- Rbitter['media_downloader']['large_image'])
39
- end
40
-
41
- def xmlrpcd_start
42
- if Rbitter['xmlrpc']['enable']
43
- @rpc_service = Thread.new {
44
- rpc_server = @xmlrpcd_class.new(Rbitter['xmlrpc']['bind_host'], Rbitter['xmlrpc']['bind_port'])
45
- rpc_server.main_loop
46
- }
47
- else
48
- @rpc_service = nil
49
- end
50
- end
51
-
52
- def xmlrpcd_stop
53
- unless @rpc_service.nil?
54
- if @rpc_service.alive?
55
- puts "Finishing RPCServer (impl: #{@xmlrpcd_class})"
56
- @rpc_service.terminate
57
- @rpc_service.join
58
- @rpc_service = nil
59
- end
60
- end
61
- end
62
-
63
- def mark(code, message)
64
- Record.create({:marker => code,
65
- :marker_msg => message,
66
- :userid => nil,
67
- :username => nil,
68
- :tweetid => nil,
69
- :replyto => nil,
70
- :tweet => nil,
71
- :date => ARSupport.any_to_datestring(DateTime.now),
72
- :rt_count => 0,
73
- :fav_count => 0})
74
- end
75
-
76
- def write_init_marker
77
- mark(LOG_INIT, "Archiving service started")
78
- end
79
-
80
- def write_halt_marker
81
- mark(LOG_HALT, "Archiving service halted")
82
- end
83
-
84
- def main_loop(streaming_adapter = Rbitter::StreamClient)
85
- xmlrpcd_start if Rbitter['xmlrpc']['enable']
86
-
87
- begin
88
- write_init_marker
89
-
90
- streaming_adapter.new(Rbitter['twitter'].dup).run { |a|
91
- @dt << a['media_urls']
92
-
93
- record = Record.find_or_initialize_by(tweetid: a['tweetid'])
94
- record.update({:marker => 0,
95
- :marker_msg => "normal",
96
- :userid => a['userid'],
97
- :username => a['screen_name'],
98
- :tweetid => a['tweetid'],
99
- :replyto => a['replyto'],
100
- :tweet => a['tweet'],
101
- :date => a['date'],
102
- :rt_count => a['rt_count'],
103
- :fav_count => a['fav_count']})
104
-
105
- record.save
106
- }
107
- rescue Interrupt => e
108
- puts ""
109
- puts "Interrupted..."
110
- rescue Twitter::Error::Unauthorized => e
111
- puts "Please configure your Twitter token on config.json."
112
- rescue Twitter::Error::ServerError, Resolv::ResolvError => e
113
- puts "Service unavailable now. Retry in 5 second..."
114
- sleep 5
115
- retry
116
- rescue Twitter::Error => e
117
- puts "Twitter Error: #{e.inspect}"
118
- ensure
119
- xmlrpcd_stop if Rbitter['xmlrpc']['enable']
120
- @dt.job_cleanup
121
-
122
- write_halt_marker
123
- end
124
- end
125
- end
126
- end
1
+ # encoding: utf-8
2
+
3
+ require "json"
4
+ require "date"
5
+ require "twitter"
6
+ require "resolv"
7
+
8
+ require "rbitter/records"
9
+ require "rbitter/streaming"
10
+ require "rbitter/dlthread"
11
+ require "rbitter/xmlrpc"
12
+
13
+ module Rbitter
14
+ class ArcServer
15
+ LOG_NORMAL = 0
16
+ LOG_INIT = 1
17
+ LOG_HALT = 2
18
+
19
+ def initialize(xmlrpcd_class = Rbitter::RPCServer)
20
+ @xmlrpcd_class = xmlrpcd_class
21
+
22
+ ARSupport.connect_database
23
+
24
+ if not ARSupport.prepared?
25
+ puts "Initiate database table..."
26
+ if Rbitter['activerecord'] == 'mysql2'
27
+ ARSupport.prepare "DEFAULT CHARSET=utf8mb4"
28
+ else
29
+ ARSupport.prepare
30
+ end
31
+ end
32
+
33
+ ARSupport.update_database_scheme
34
+
35
+ @dt = DLThread.new(
36
+ Rbitter['media_downloader']['download_dir'],
37
+ Rbitter['media_downloader']['large_image'])
38
+ end
39
+
40
+ def xmlrpcd_start
41
+ if Rbitter['xmlrpc']['enable']
42
+ @rpc_service = Thread.new {
43
+ rpc_server = @xmlrpcd_class.new(Rbitter['xmlrpc']['bind_host'], Rbitter['xmlrpc']['bind_port'])
44
+ rpc_server.main_loop
45
+ }
46
+ else
47
+ @rpc_service = nil
48
+ end
49
+ end
50
+
51
+ def xmlrpcd_stop
52
+ unless @rpc_service.nil?
53
+ if @rpc_service.alive?
54
+ puts "Finishing RPCServer (impl: #{@xmlrpcd_class})"
55
+ @rpc_service.terminate
56
+ @rpc_service.join
57
+ @rpc_service = nil
58
+ end
59
+ end
60
+ end
61
+
62
+ def mark(code, message)
63
+ Record.create({:marker => code,
64
+ :marker_msg => message,
65
+ :userid => nil,
66
+ :username => nil,
67
+ :tweetid => nil,
68
+ :replyto => nil,
69
+ :tweet => nil,
70
+ :date => ARSupport.any_to_datestring(DateTime.now),
71
+ :rt_count => 0,
72
+ :fav_count => 0})
73
+ end
74
+
75
+ def write_init_marker
76
+ mark(LOG_INIT, "Archiving service started")
77
+ end
78
+
79
+ def write_halt_marker
80
+ mark(LOG_HALT, "Archiving service halted")
81
+ end
82
+
83
+ def main_loop(streaming_adapter = Rbitter::StreamClient)
84
+ xmlrpcd_start if Rbitter['xmlrpc']['enable']
85
+
86
+ begin
87
+ write_init_marker
88
+
89
+ streaming_adapter.new(Rbitter['twitter'].dup).run { |a|
90
+ @dt << a['media_urls']
91
+
92
+ record = Record.find_or_initialize_by(tweetid: a['tweetid'])
93
+ record.update({:marker => 0,
94
+ :marker_msg => "normal",
95
+ :userid => a['userid'],
96
+ :username => a['screen_name'],
97
+ :tweetid => a['tweetid'],
98
+ :replyto => a['replyto'],
99
+ :tweet => a['tweet'],
100
+ :date => a['date'],
101
+ :rt_count => a['rt_count'],
102
+ :fav_count => a['fav_count']})
103
+
104
+ record.save
105
+ }
106
+ rescue Interrupt => e
107
+ puts ""
108
+ puts "Interrupted..."
109
+ rescue Twitter::Error::Unauthorized => e
110
+ warn "Twitter access unauthorized:"
111
+ warn " Possible solutions"
112
+ warn " 1. Configure Twitter token on config.json"
113
+ warn " 2. Check system time (Time is important on authentication)"
114
+ warn " 3. Check Twitter account status"
115
+ rescue Twitter::Error::ServerError, Resolv::ResolvError => e
116
+ puts "Service unavailable now. Retry in 5 second..."
117
+ sleep 5
118
+ retry
119
+ rescue Twitter::Error => e
120
+ warn "Twitter Error: #{e.inspect}"
121
+ warn "Main loop of ArcServer halted."
122
+ ensure
123
+ xmlrpcd_stop if Rbitter['xmlrpc']['enable']
124
+ @dt.job_cleanup
125
+
126
+ write_halt_marker
127
+ end
128
+ end
129
+ end
130
+ end