rbitter 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +21 -21
  3. data/.rspec +2 -2
  4. data/.travis.yml +15 -15
  5. data/Gemfile +12 -12
  6. data/LICENSE.txt +22 -22
  7. data/Rakefile +8 -8
  8. data/XMLRPC.md +19 -19
  9. data/bin/rbitter +20 -20
  10. data/lib/rbitter/arcserver.rb +130 -126
  11. data/lib/rbitter/console.rb +93 -93
  12. data/lib/rbitter/default/config_json.rb +37 -38
  13. data/lib/rbitter/dlthread.rb +62 -65
  14. data/lib/rbitter/env.rb +62 -62
  15. data/lib/rbitter/libtwitter_connection_override.rb +45 -45
  16. data/lib/rbitter/records.rb +121 -121
  17. data/lib/rbitter/records_migrate/20150327_add_index.rb +11 -11
  18. data/lib/rbitter/records_migrate/20150504_add_replyto_column.rb +11 -11
  19. data/lib/rbitter/streaming.rb +104 -104
  20. data/lib/rbitter/version.rb +20 -20
  21. data/lib/rbitter/xmlrpc.rb +3 -3
  22. data/lib/rbitter/xmlrpcd/base.rb +24 -24
  23. data/lib/rbitter/xmlrpcd/rpchandles.rb +11 -11
  24. data/lib/rbitter/xmlrpcd/xmlrpc_auth_server.rb +82 -82
  25. data/lib/rbitter/xmlrpcd/xmlrpcd.rb +69 -69
  26. data/lib/rbitter.rb +86 -86
  27. data/rbitter.gemspec +46 -42
  28. data/spec/config/default.json +32 -32
  29. data/spec/rbitter/arcserver_spec.rb +30 -30
  30. data/spec/rbitter/console_spec.rb +9 -9
  31. data/spec/rbitter/default/config_json_spec.rb +3 -3
  32. data/spec/rbitter/dlthread_spec.rb +8 -13
  33. data/spec/rbitter/env_spec.rb +62 -62
  34. data/spec/rbitter/libtwitter_connection_override_spec.rb +8 -8
  35. data/spec/rbitter/records_spec.rb +13 -13
  36. data/spec/rbitter/streaming_spec.rb +9 -9
  37. data/spec/rbitter/version_spec.rb +8 -8
  38. data/spec/rbitter/xmlrpc_spec.rb +8 -8
  39. data/spec/rbitter/xmlrpcd/base_spec.rb +29 -29
  40. data/spec/rbitter/xmlrpcd/rpchandles_spec.rb +10 -10
  41. data/spec/rbitter/xmlrpcd/xmlrpc_auth_server_spec.rb +8 -8
  42. data/spec/rbitter/xmlrpcd/xmlrpcd_spec.rb +9 -9
  43. data/spec/rbitter_spec.rb +42 -42
  44. data/spec/spec_helper.rb +39 -39
  45. metadata +29 -56
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 44878b7491619cb57321ab471e8b03279180932b
4
- data.tar.gz: a1dbfc5622f0d65b9aa256992a18b8523d425b0d
3
+ metadata.gz: 3ca2dd0e8a08659416b4e1ba154375673bfe697d
4
+ data.tar.gz: d60daa351fef6fdfa8cc3fdb444b31e10a820c80
5
5
  SHA512:
6
- metadata.gz: d82fc7a0b2158f54aacd644d31aad960614f89615b0c4b471fab30d3fd304627cb2f72aae757d141af4bc4267c6bbf7d0140b68f249833a85bdcebd346feead1
7
- data.tar.gz: a7c43429c676d18247d8fb274956e289ef537b7fe832f8240f869c0748263e36b6e14ab3702bc72e2b81dd5fec9723a83c8de25e50d8ce4214be7bf14027857e
6
+ metadata.gz: 251d4bda5d89583701ca2ecb3b52c5cafeab42b6e3f01d90692e2457d48f53e22ff9cda790daed72b63088cd077c4ba7ba27e8dd0751e9d2b5f1dbbf68dd1286
7
+ data.tar.gz: cb04a2013000e2581b5b8c6dd5b30b15058c6f70f995fbde2dbf31733e78a23c7b8526cafc203cae9d0125f16752c8728cc45b3103f28a83e2d52dbac153118e
data/.gitignore CHANGED
@@ -1,21 +1,21 @@
1
- /.bundle/
2
- /.yardoc
3
- /Gemfile.lock
4
- /_yardoc/
5
- /coverage/
6
- /doc/
7
- /pkg/
8
- /spec/reports/
9
- /tmp/
10
- /build/
11
- /vendor/
12
- *.bundle
13
- *.so
14
- *.o
15
- *.a
16
- mkmf.log
17
- *.gem
18
- *.json
19
- *.sqlite
20
- .rbitter
21
- .rbitter/*
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
10
+ /build/
11
+ /vendor/
12
+ *.bundle
13
+ *.so
14
+ *.o
15
+ *.a
16
+ mkmf.log
17
+ *.gem
18
+ *.json
19
+ *.sqlite
20
+ .rbitter
21
+ .rbitter/*
data/.rspec CHANGED
@@ -1,2 +1,2 @@
1
- --color
2
- --require spec_helper
1
+ --color
2
+ --require spec_helper
data/.travis.yml CHANGED
@@ -1,15 +1,15 @@
1
- language: ruby
2
- rvm:
3
- - 1.9.3
4
- - 2.1.0
5
- - 2.2.0
6
- - jruby-19mode
7
- - jruby-head
8
- - rbx-2
9
-
10
- env: LANG="en_US.UTF-8" LC_ALL="en_US.UTF-8"
11
-
12
- matrix:
13
- allow_failures:
14
- - rvm: ruby-head
15
- - rvm: rbx-2
1
+ language: ruby
2
+ rvm:
3
+ - 1.9.3
4
+ - 2.1.0
5
+ - 2.2.0
6
+ - jruby-19mode
7
+ - jruby-head
8
+ - rbx-2
9
+
10
+ env: LANG="en_US.UTF-8" LC_ALL="en_US.UTF-8"
11
+
12
+ matrix:
13
+ allow_failures:
14
+ - rvm: ruby-head
15
+ - rvm: rbx-2
data/Gemfile CHANGED
@@ -1,12 +1,12 @@
1
- source "https://rubygems.org"
2
-
3
- group :test do
4
- gem 'rspec', '~>3.0'
5
- gem 'simplecov', :require => false, :group => :test
6
- gem 'coveralls', :require => false
7
- end
8
-
9
- gem "bundler"
10
- gem 'jruby-openssl', platforms: :jruby
11
-
12
- gemspec
1
+ source "https://rubygems.org"
2
+
3
+ group :test do
4
+ gem 'rspec', '~>3.0'
5
+ gem 'simplecov', :require => false, :group => :test
6
+ gem 'coveralls', :require => false
7
+ end
8
+
9
+ gem "bundler"
10
+ gem 'jruby-openssl', platforms: :jruby
11
+
12
+ gemspec
data/LICENSE.txt CHANGED
@@ -1,22 +1,22 @@
1
- Copyright (c) 2015 Nidev Plontra
2
-
3
- MIT License
4
-
5
- Permission is hereby granted, free of charge, to any person obtaining
6
- a copy of this software and associated documentation files (the
7
- "Software"), to deal in the Software without restriction, including
8
- without limitation the rights to use, copy, modify, merge, publish,
9
- distribute, sublicense, and/or sell copies of the Software, and to
10
- permit persons to whom the Software is furnished to do so, subject to
11
- the following conditions:
12
-
13
- The above copyright notice and this permission notice shall be
14
- included in all copies or substantial portions of the Software.
15
-
16
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
- LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
- OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
- WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
1
+ Copyright (c) 2015 Nidev Plontra
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/Rakefile CHANGED
@@ -1,8 +1,8 @@
1
- require "bundler/gem_tasks"
2
- require "rspec/core/rake_task"
3
-
4
- RSpec::Core::RakeTask.new
5
-
6
- task :default => :spec
7
- task :basic_test => :spec
8
-
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+
4
+ RSpec::Core::RakeTask.new
5
+
6
+ task :default => :spec
7
+ task :basic_test => :spec
8
+
data/XMLRPC.md CHANGED
@@ -1,19 +1,19 @@
1
- # What is RPC handle ? #
2
- ## Commands ##
3
- ### Authentication ###
4
- ### Revoke Authentication Token ###
5
- ### Echo ###
6
- ### Last Active ###
7
- ### Retriever ###
8
- ### Statistic ###
9
- # How to write own RPC handle? #
10
- RPC handle is a Ruby class. Writing a method in Ruby class, that's it. Names of methods are treated as XMLRPC command.
11
-
12
- When you write a new class for your own RPC handle, you must inherit either Auth or NoAuth class from rpc/base.rb.
13
-
14
- * class Auth < Object: Methods in a Ruby class inheriting Auth requires *auth_key* to access.
15
- * class NoAuth < Object: Methods in a Ruby class inheriting NoAuth doesn't require *auth_key* and these XMLRPC commands can be called by anonymous user.
16
-
17
- Filename should start with 'rh_'. It's prefix to be autoloaded by xmlrpc.rb.
18
-
19
- Refer rpc/rh_echo.rb as an example.
1
+ # What is RPC handle ? #
2
+ ## Commands ##
3
+ ### Authentication ###
4
+ ### Revoke Authentication Token ###
5
+ ### Echo ###
6
+ ### Last Active ###
7
+ ### Retriever ###
8
+ ### Statistic ###
9
+ # How to write own RPC handle? #
10
+ RPC handle is a Ruby class. Writing a method in Ruby class, that's it. Names of methods are treated as XMLRPC command.
11
+
12
+ When you write a new class for your own RPC handle, you must inherit either Auth or NoAuth class from rpc/base.rb.
13
+
14
+ * class Auth < Object: Methods in a Ruby class inheriting Auth requires *auth_key* to access.
15
+ * class NoAuth < Object: Methods in a Ruby class inheriting NoAuth doesn't require *auth_key* and these XMLRPC commands can be called by anonymous user.
16
+
17
+ Filename should start with 'rh_'. It's prefix to be autoloaded by xmlrpc.rb.
18
+
19
+ Refer rpc/rh_echo.rb as an example.
data/bin/rbitter CHANGED
@@ -1,20 +1,20 @@
1
- #!/usr/bin/env ruby
2
-
3
- require "rbitter"
4
-
5
- Rbitter.rbitter_header
6
-
7
- if ARGV.length < 1 || (not Rbitter::BOOTSTRAP_ARGS.include?(ARGV[0]))
8
- Rbitter.rbitter_help_msg
9
- exit -1
10
- else
11
- Rbitter.bootstrap(ARGV)
12
-
13
- if ARGV[0] == "configure"
14
- puts "Writing config.json done"
15
- puts "You can put config.json one of these locations:"
16
- puts "[1] config.json (current folder)"
17
- puts "[2] .rbitter/config.json (current folder)"
18
- end
19
- end
20
-
1
+ #!/usr/bin/env ruby
2
+
3
+ require "rbitter"
4
+
5
+ Rbitter.rbitter_header
6
+
7
+ if ARGV.length < 1 || (not Rbitter::BOOTSTRAP_ARGS.include?(ARGV[0]))
8
+ Rbitter.rbitter_help_msg
9
+ exit -1
10
+ else
11
+ Rbitter.bootstrap(ARGV)
12
+
13
+ if ARGV[0] == "configure"
14
+ puts "Writing config.json done"
15
+ puts "You can put config.json one of these locations:"
16
+ puts "[1] config.json (current folder)"
17
+ puts "[2] .rbitter/config.json (current folder)"
18
+ end
19
+ end
20
+
@@ -1,126 +1,130 @@
1
- # encoding: utf-8
2
-
3
- require "json"
4
- require "date"
5
- require "twitter"
6
- require "resolv"
7
-
8
- require "rbitter/records"
9
- require "rbitter/streaming"
10
- require "rbitter/dlthread"
11
- require "rbitter/xmlrpc"
12
-
13
- module Rbitter
14
- class ArcServer
15
- LOG_NORMAL = 0
16
- LOG_INIT = 1
17
- LOG_HALT = 2
18
-
19
- def initialize(xmlrpcd_class = Rbitter::RPCServer)
20
- @xmlrpcd_class = xmlrpcd_class
21
-
22
- ARSupport.connect_database
23
-
24
- if not ARSupport.prepared?
25
- puts "Initiate database table..."
26
- if Rbitter['activerecord'] == 'mysql2'
27
- ARSupport.prepare "DEFAULT CHARSET=utf8mb4"
28
- else
29
- ARSupport.prepare
30
- end
31
- end
32
-
33
- ARSupport.update_database_scheme
34
-
35
- @dt = DLThread.new(
36
- Rbitter['media_downloader']['download_dir'],
37
- Rbitter['media_downloader']['cacert_path'],
38
- Rbitter['media_downloader']['large_image'])
39
- end
40
-
41
- def xmlrpcd_start
42
- if Rbitter['xmlrpc']['enable']
43
- @rpc_service = Thread.new {
44
- rpc_server = @xmlrpcd_class.new(Rbitter['xmlrpc']['bind_host'], Rbitter['xmlrpc']['bind_port'])
45
- rpc_server.main_loop
46
- }
47
- else
48
- @rpc_service = nil
49
- end
50
- end
51
-
52
- def xmlrpcd_stop
53
- unless @rpc_service.nil?
54
- if @rpc_service.alive?
55
- puts "Finishing RPCServer (impl: #{@xmlrpcd_class})"
56
- @rpc_service.terminate
57
- @rpc_service.join
58
- @rpc_service = nil
59
- end
60
- end
61
- end
62
-
63
- def mark(code, message)
64
- Record.create({:marker => code,
65
- :marker_msg => message,
66
- :userid => nil,
67
- :username => nil,
68
- :tweetid => nil,
69
- :replyto => nil,
70
- :tweet => nil,
71
- :date => ARSupport.any_to_datestring(DateTime.now),
72
- :rt_count => 0,
73
- :fav_count => 0})
74
- end
75
-
76
- def write_init_marker
77
- mark(LOG_INIT, "Archiving service started")
78
- end
79
-
80
- def write_halt_marker
81
- mark(LOG_HALT, "Archiving service halted")
82
- end
83
-
84
- def main_loop(streaming_adapter = Rbitter::StreamClient)
85
- xmlrpcd_start if Rbitter['xmlrpc']['enable']
86
-
87
- begin
88
- write_init_marker
89
-
90
- streaming_adapter.new(Rbitter['twitter'].dup).run { |a|
91
- @dt << a['media_urls']
92
-
93
- record = Record.find_or_initialize_by(tweetid: a['tweetid'])
94
- record.update({:marker => 0,
95
- :marker_msg => "normal",
96
- :userid => a['userid'],
97
- :username => a['screen_name'],
98
- :tweetid => a['tweetid'],
99
- :replyto => a['replyto'],
100
- :tweet => a['tweet'],
101
- :date => a['date'],
102
- :rt_count => a['rt_count'],
103
- :fav_count => a['fav_count']})
104
-
105
- record.save
106
- }
107
- rescue Interrupt => e
108
- puts ""
109
- puts "Interrupted..."
110
- rescue Twitter::Error::Unauthorized => e
111
- puts "Please configure your Twitter token on config.json."
112
- rescue Twitter::Error::ServerError, Resolv::ResolvError => e
113
- puts "Service unavailable now. Retry in 5 second..."
114
- sleep 5
115
- retry
116
- rescue Twitter::Error => e
117
- puts "Twitter Error: #{e.inspect}"
118
- ensure
119
- xmlrpcd_stop if Rbitter['xmlrpc']['enable']
120
- @dt.job_cleanup
121
-
122
- write_halt_marker
123
- end
124
- end
125
- end
126
- end
1
+ # encoding: utf-8
2
+
3
+ require "json"
4
+ require "date"
5
+ require "twitter"
6
+ require "resolv"
7
+
8
+ require "rbitter/records"
9
+ require "rbitter/streaming"
10
+ require "rbitter/dlthread"
11
+ require "rbitter/xmlrpc"
12
+
13
+ module Rbitter
14
+ class ArcServer
15
+ LOG_NORMAL = 0
16
+ LOG_INIT = 1
17
+ LOG_HALT = 2
18
+
19
+ def initialize(xmlrpcd_class = Rbitter::RPCServer)
20
+ @xmlrpcd_class = xmlrpcd_class
21
+
22
+ ARSupport.connect_database
23
+
24
+ if not ARSupport.prepared?
25
+ puts "Initiate database table..."
26
+ if Rbitter['activerecord'] == 'mysql2'
27
+ ARSupport.prepare "DEFAULT CHARSET=utf8mb4"
28
+ else
29
+ ARSupport.prepare
30
+ end
31
+ end
32
+
33
+ ARSupport.update_database_scheme
34
+
35
+ @dt = DLThread.new(
36
+ Rbitter['media_downloader']['download_dir'],
37
+ Rbitter['media_downloader']['large_image'])
38
+ end
39
+
40
+ def xmlrpcd_start
41
+ if Rbitter['xmlrpc']['enable']
42
+ @rpc_service = Thread.new {
43
+ rpc_server = @xmlrpcd_class.new(Rbitter['xmlrpc']['bind_host'], Rbitter['xmlrpc']['bind_port'])
44
+ rpc_server.main_loop
45
+ }
46
+ else
47
+ @rpc_service = nil
48
+ end
49
+ end
50
+
51
+ def xmlrpcd_stop
52
+ unless @rpc_service.nil?
53
+ if @rpc_service.alive?
54
+ puts "Finishing RPCServer (impl: #{@xmlrpcd_class})"
55
+ @rpc_service.terminate
56
+ @rpc_service.join
57
+ @rpc_service = nil
58
+ end
59
+ end
60
+ end
61
+
62
+ def mark(code, message)
63
+ Record.create({:marker => code,
64
+ :marker_msg => message,
65
+ :userid => nil,
66
+ :username => nil,
67
+ :tweetid => nil,
68
+ :replyto => nil,
69
+ :tweet => nil,
70
+ :date => ARSupport.any_to_datestring(DateTime.now),
71
+ :rt_count => 0,
72
+ :fav_count => 0})
73
+ end
74
+
75
+ def write_init_marker
76
+ mark(LOG_INIT, "Archiving service started")
77
+ end
78
+
79
+ def write_halt_marker
80
+ mark(LOG_HALT, "Archiving service halted")
81
+ end
82
+
83
+ def main_loop(streaming_adapter = Rbitter::StreamClient)
84
+ xmlrpcd_start if Rbitter['xmlrpc']['enable']
85
+
86
+ begin
87
+ write_init_marker
88
+
89
+ streaming_adapter.new(Rbitter['twitter'].dup).run { |a|
90
+ @dt << a['media_urls']
91
+
92
+ record = Record.find_or_initialize_by(tweetid: a['tweetid'])
93
+ record.update({:marker => 0,
94
+ :marker_msg => "normal",
95
+ :userid => a['userid'],
96
+ :username => a['screen_name'],
97
+ :tweetid => a['tweetid'],
98
+ :replyto => a['replyto'],
99
+ :tweet => a['tweet'],
100
+ :date => a['date'],
101
+ :rt_count => a['rt_count'],
102
+ :fav_count => a['fav_count']})
103
+
104
+ record.save
105
+ }
106
+ rescue Interrupt => e
107
+ puts ""
108
+ puts "Interrupted..."
109
+ rescue Twitter::Error::Unauthorized => e
110
+ warn "Twitter access unauthorized:"
111
+ warn " Possible solutions"
112
+ warn " 1. Configure Twitter token on config.json"
113
+ warn " 2. Check system time (Time is important on authentication)"
114
+ warn " 3. Check Twitter account status"
115
+ rescue Twitter::Error::ServerError, Resolv::ResolvError => e
116
+ puts "Service unavailable now. Retry in 5 second..."
117
+ sleep 5
118
+ retry
119
+ rescue Twitter::Error => e
120
+ warn "Twitter Error: #{e.inspect}"
121
+ warn "Main loop of ArcServer halted."
122
+ ensure
123
+ xmlrpcd_stop if Rbitter['xmlrpc']['enable']
124
+ @dt.job_cleanup
125
+
126
+ write_halt_marker
127
+ end
128
+ end
129
+ end
130
+ end