rbitter 0.2.1-java → 0.2.3-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +0 -1
  3. data/lib/rbitter/arcserver.rb +3 -2
  4. data/lib/rbitter/dlthread.rb +29 -22
  5. data/lib/rbitter/env.rb +5 -5
  6. data/lib/rbitter/progress.rb +4 -1
  7. data/lib/rbitter/records.rb +4 -1
  8. data/lib/rbitter/version.rb +1 -1
  9. data/rbitter.gemspec +2 -2
  10. metadata +47 -32
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 06bfae2b2f77d2fc7187c107ddcb42f3b8240860
4
- data.tar.gz: 6956d800f7d306b1ae4b6b1b78b3a4e1c383e954
3
+ metadata.gz: cc1c405538b3bf82c8ee56ab921ac307e95b9b92
4
+ data.tar.gz: 0134c8c6118a264f544509b23a4a113a46eeb511
5
5
  SHA512:
6
- metadata.gz: 8d745a88bc36645e6e3b2b2224a192a44e4ca6dfb09f136b82ba275bdcc29abe2fe01ff7c51bd9dd9d8d1187eaab56b58e66cf8977c4da625b2c8c547036efb9
7
- data.tar.gz: 1676d89e1c4b4fdb7b24a3d0f6d9263148d327fb82547de298dc7ed81f734df502a734bd083b61b3c003e59719b6867cf19e25a46a4be78ab0f5ee667b0962bb
6
+ metadata.gz: f41ac58c4de23d27cc99b6a6a2627a3da1225f69550be1c2571dbe29028d4d95eaf2b7bf07df29320e4fae570c3b6fc461492b1a6b2ef2fc2c282c13597e8fe1
7
+ data.tar.gz: d2bc79071ee6f265abb790de90f5059a7761b32f2e023d40fc6418a20cfe8e46cbb505d7eba3e1511a3f26128e4b8a861727a6e3479f719e963040ccc54af028
data/Gemfile CHANGED
@@ -1,7 +1,6 @@
1
1
  source "https://rubygems.org"
2
2
 
3
3
  group :test do
4
- gem 'rspec', '~>3.0'
5
4
  gem 'simplecov', :require => false, :group => :test
6
5
  gem 'coveralls', :require => false
7
6
  end
data/lib/rbitter/arcserver.rb CHANGED
@@ -114,6 +114,7 @@ module Rbitter
114
114
 
115
115
  streaming_adapter.new(Rbitter['twitter']).run { |a|
116
116
  @dt << a['media_urls']
117
+ draw "[rbitter] saving @#{a['screen_name']}:#{a['tweetid']}"
117
118
 
118
119
  record = Record.find_or_initialize_by(tweetid: a['tweetid'])
119
120
  record.update({:marker => 0,
@@ -128,7 +129,7 @@ module Rbitter
128
129
  :fav_count => a['fav_count']})
129
130
 
130
131
  record.save
131
- draw "[rbitter] saving tweet: #{a['tweetid']}"
132
+ draw "[rbitter] saved @#{a['screen_name']}:#{a['tweetid']}"
132
133
  }
133
134
  rescue Interrupt => e
134
135
  puts ""
@@ -146,7 +147,7 @@ module Rbitter
146
147
 
147
148
  retry if resurrect_loop?
148
149
  rescue Resolv::ResolvError, Errno::ECONNABORTED,
149
- Errno::ECONNREFUSED, Errno::ECONNRESET => e
150
+ Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ETIMEDOUT => e
150
151
  puts "Network problem. Retry in 5 seconds..."
151
152
  mark_error(e.to_s, "(retry) Network problem")
152
153
 
data/lib/rbitter/dlthread.rb CHANGED
@@ -5,6 +5,8 @@ require "openssl"
5
5
 
6
6
  module Rbitter
7
7
  class DLThread
8
+ MAX_THREADS = 20
9
+
8
10
  def initialize(dlfolder, large_flag)
9
11
  @dest = dlfolder
10
12
  if not File.directory?(dlfolder)
@@ -23,27 +25,34 @@ module Rbitter
23
25
  end
24
26
 
25
27
  def <<(url_array)
26
- download_task = Thread.new {
28
+ if @pool.length >= MAX_THREADS
29
+ job_cleanup
30
+ end
31
+
32
+ download_task = Thread.new do
27
33
  url_array.each { |url|
28
34
  uri = URI.parse(@large_image ? url + ":large" : url)
29
- ssl = uri.scheme.downcase == 'https'
30
-
31
- Net::HTTP.start(uri.host, uri.port, :use_ssl => ssl) { |h|
32
- req = Net::HTTP::Get.new uri.request_uri
33
- h.request(req) { |res|
34
- case res
35
- when Net::HTTPOK
36
- fname = File.basename(url)
37
-
38
- puts "[fetch] remote: #{uri.path} => local: #{fname}"
39
- open(File.join(@dest, fname), "wb") { |file|
40
- res.read_body { |chunk| file.write(chunk) }
41
- }
42
- end
35
+
36
+ begin
37
+ Net::HTTP.start(uri.host, uri.port, :use_ssl => uri.scheme.downcase == 'https') { |h|
38
+ req = Net::HTTP::Get.new uri.request_uri
39
+ h.request(req) { |res|
40
+ case res
41
+ when Net::HTTPOK
42
+ fname = File.basename(url)
43
+
44
+ puts "[fetch] remote: #{uri.path} => local: #{fname}"
45
+ open(File.join(@dest, fname), "wb") { |file|
46
+ res.read_body { |chunk| file.write(chunk) }
47
+ }
48
+ end
49
+ }
43
50
  }
44
- }
51
+ rescue => e
52
+ puts "[dlthread] exception from thread -> #{e.to_s}"
53
+ end
45
54
  }
46
- }
55
+ end
47
56
 
48
57
  @pool.push download_task
49
58
  end
@@ -52,12 +61,10 @@ module Rbitter
52
61
  until @pool.empty?
53
62
  dlthrd = @pool.shift
54
63
 
55
- if dlthrd.alive?
56
- puts "[dlthread] Thread forceful cleaning up [remains: #{@pool.length}]"
57
- dlthrd.terminate
58
- dlthrd.join
64
+ if dlthrd.join(5).nil?
65
+ puts "[dlthread] #{dlthrd.to_s} is still running. (timeout: 5sec)"
59
66
  end
60
67
  end
61
68
  end
62
69
  end
63
- end
70
+ end
data/lib/rbitter/env.rb CHANGED
@@ -63,7 +63,7 @@ module Rbitter
63
63
  # Level: warning since it is not utilized at all.
64
64
  currents.each { |conf|
65
65
  unless defaults.include?(conf)
66
- warn "[config.json] Unused config: #{conf}. You can safely remove it."
66
+ warn "[config.json] Unsupported field: #{conf}"
67
67
  end
68
68
  }
69
69
 
@@ -71,7 +71,7 @@ module Rbitter
71
71
  # Level: error and program should stop. (return false for this)
72
72
  defaults.each { |conf|
73
73
  unless currents.include?(conf)
74
- warn "[config.json] Config not found: #{conf}. Invalid configuration!"
74
+ warn "[config.json] Error! Missing field: #{conf}"
75
75
  not_errored = false
76
76
  end
77
77
  }
@@ -88,9 +88,9 @@ module Rbitter
88
88
  }
89
89
 
90
90
  return @@env if env_valid?
91
- fail StandardError, "Invalid configuration"
91
+ fail StandardError, "Void configuration. Maybe failure on load?"
92
92
  rescue => e
93
- fail ConfigFileError, "Load Failure (#{json_path}): #{e.to_s}"
93
+ fail ConfigFileError, "Config error on (#{json_path}): #{e.to_s}"
94
94
  end
95
95
  end
96
96
 
@@ -111,6 +111,6 @@ module Rbitter
111
111
  fail ConfigFileError, "No config.json on #{locations.join(' or ')}" if @@env.empty?
112
112
  fail ConfigFileError, "Configuration outdated. Please see above messages to update it" if not env_valid?
113
113
 
114
- puts "[config.json] Loaded configuration is valid. good to go!"
114
+ puts "[config.json] JSON data structure is safe to go."
115
115
  end
116
116
  end
data/lib/rbitter/progress.rb CHANGED
@@ -10,7 +10,10 @@ module Rbitter
10
10
  def draw sentence
11
11
  $> << sentence
12
12
  if sentence.length < @@last_draw
13
- $> << " "*(@@last_draw - sentence.length)
13
+ clear_char_len = @@last_draw - sentence.length
14
+ clear_char_len.times {
15
+ $> << " "
16
+ }
14
17
  end
15
18
  @@last_draw = sentence.length
16
19
  putback
data/lib/rbitter/records.rb CHANGED
@@ -47,7 +47,10 @@ module ARSupport
47
47
  timeout: 10000) # Long timeout for slow computer
48
48
  end
49
49
  elsif Rbitter['activerecord'] == 'mysql2'
50
- Jdbc::MySQL.load_driver if RUBY_PLATFORM == 'java'
50
+ if RUBY_PLATFORM == 'java'
51
+ require 'jdbc/mysql'
52
+ Jdbc::MySQL.load_driver
53
+ end
51
54
 
52
55
  ActiveRecord::Base.establish_connection(
53
56
  adapter: (RUBY_PLATFORM == 'java' ? 'jdbcmysql' : 'mysql2'),
data/lib/rbitter/version.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  module Rbitter
2
2
  PRODUCT_NAME = "Rbitter"
3
- VERSION = "0.2.1"
3
+ VERSION = "0.2.3"
4
4
 
5
5
  def major
6
6
  VERSION.match(/^([0-9]+)\./)[1]
data/rbitter.gemspec CHANGED
@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
23
23
  spec.add_dependency 'twitter', '~> 5.14'
24
24
  spec.add_dependency 'json', '~> 1.7'
25
25
  spec.add_dependency 'ripl', '~> 0.7'
26
- spec.add_dependency 'activerecord', '~> 4.0'
26
+ spec.add_dependency 'activerecord', '~> 4.2'
27
27
 
28
28
  if RUBY_PLATFORM == 'java'
29
29
  spec.platform = 'java'
@@ -38,9 +38,9 @@ Gem::Specification.new do |spec|
38
38
 
39
39
  spec.add_dependency 'sqlite3', '~> 1.3'
40
40
  spec.add_dependency 'mysql2', '~> 0.3'
41
- spec.add_dependency 'activerecord-mysql2-adapter', '~> 0.0.3'
42
41
  end
43
42
 
44
43
  spec.add_development_dependency "bundler", "~> 1.6"
45
44
  spec.add_development_dependency "rake", "~> 10.0"
45
+ spec.add_development_dependency "rspec", "~> 3.3"
46
46
  end
metadata CHANGED
@@ -1,19 +1,19 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rbitter
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.1
4
+ version: 0.2.3
5
5
  platform: java
6
6
  authors:
7
7
  - Nidev Plontra
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-06-20 00:00:00.000000000 Z
11
+ date: 2015-08-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
15
15
  requirements:
16
- - - ~>
16
+ - - "~>"
17
17
  - !ruby/object:Gem::Version
18
18
  version: '5.14'
19
19
  name: twitter
@@ -21,13 +21,13 @@ dependencies:
21
21
  type: :runtime
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ~>
24
+ - - "~>"
25
25
  - !ruby/object:Gem::Version
26
26
  version: '5.14'
27
27
  - !ruby/object:Gem::Dependency
28
28
  requirement: !ruby/object:Gem::Requirement
29
29
  requirements:
30
- - - ~>
30
+ - - "~>"
31
31
  - !ruby/object:Gem::Version
32
32
  version: '1.7'
33
33
  name: json
@@ -35,13 +35,13 @@ dependencies:
35
35
  type: :runtime
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - ~>
38
+ - - "~>"
39
39
  - !ruby/object:Gem::Version
40
40
  version: '1.7'
41
41
  - !ruby/object:Gem::Dependency
42
42
  requirement: !ruby/object:Gem::Requirement
43
43
  requirements:
44
- - - ~>
44
+ - - "~>"
45
45
  - !ruby/object:Gem::Version
46
46
  version: '0.7'
47
47
  name: ripl
@@ -49,27 +49,27 @@ dependencies:
49
49
  type: :runtime
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - ~>
52
+ - - "~>"
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0.7'
55
55
  - !ruby/object:Gem::Dependency
56
56
  requirement: !ruby/object:Gem::Requirement
57
57
  requirements:
58
- - - ~>
58
+ - - "~>"
59
59
  - !ruby/object:Gem::Version
60
- version: '4.0'
60
+ version: '4.2'
61
61
  name: activerecord
62
62
  prerelease: false
63
63
  type: :runtime
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - ~>
66
+ - - "~>"
67
67
  - !ruby/object:Gem::Version
68
- version: '4.0'
68
+ version: '4.2'
69
69
  - !ruby/object:Gem::Dependency
70
70
  requirement: !ruby/object:Gem::Requirement
71
71
  requirements:
72
- - - ~>
72
+ - - "~>"
73
73
  - !ruby/object:Gem::Version
74
74
  version: '1.3'
75
75
  name: activerecord-jdbc-adapter
@@ -77,13 +77,13 @@ dependencies:
77
77
  type: :runtime
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
- - - ~>
80
+ - - "~>"
81
81
  - !ruby/object:Gem::Version
82
82
  version: '1.3'
83
83
  - !ruby/object:Gem::Dependency
84
84
  requirement: !ruby/object:Gem::Requirement
85
85
  requirements:
86
- - - ~>
86
+ - - "~>"
87
87
  - !ruby/object:Gem::Version
88
88
  version: '3.8'
89
89
  name: jdbc-sqlite3
@@ -91,13 +91,13 @@ dependencies:
91
91
  type: :runtime
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
- - - ~>
94
+ - - "~>"
95
95
  - !ruby/object:Gem::Version
96
96
  version: '3.8'
97
97
  - !ruby/object:Gem::Dependency
98
98
  requirement: !ruby/object:Gem::Requirement
99
99
  requirements:
100
- - - ~>
100
+ - - "~>"
101
101
  - !ruby/object:Gem::Version
102
102
  version: '5.1'
103
103
  name: jdbc-mysql
@@ -105,13 +105,13 @@ dependencies:
105
105
  type: :runtime
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
- - - ~>
108
+ - - "~>"
109
109
  - !ruby/object:Gem::Version
110
110
  version: '5.1'
111
111
  - !ruby/object:Gem::Dependency
112
112
  requirement: !ruby/object:Gem::Requirement
113
113
  requirements:
114
- - - ~>
114
+ - - "~>"
115
115
  - !ruby/object:Gem::Version
116
116
  version: '1.3'
117
117
  name: activerecord-jdbcsqlite3-adapter
@@ -119,13 +119,13 @@ dependencies:
119
119
  type: :runtime
120
120
  version_requirements: !ruby/object:Gem::Requirement
121
121
  requirements:
122
- - - ~>
122
+ - - "~>"
123
123
  - !ruby/object:Gem::Version
124
124
  version: '1.3'
125
125
  - !ruby/object:Gem::Dependency
126
126
  requirement: !ruby/object:Gem::Requirement
127
127
  requirements:
128
- - - ~>
128
+ - - "~>"
129
129
  - !ruby/object:Gem::Version
130
130
  version: '1.3'
131
131
  name: activerecord-jdbcmysql-adapter
@@ -133,13 +133,13 @@ dependencies:
133
133
  type: :runtime
134
134
  version_requirements: !ruby/object:Gem::Requirement
135
135
  requirements:
136
- - - ~>
136
+ - - "~>"
137
137
  - !ruby/object:Gem::Version
138
138
  version: '1.3'
139
139
  - !ruby/object:Gem::Dependency
140
140
  requirement: !ruby/object:Gem::Requirement
141
141
  requirements:
142
- - - ~>
142
+ - - "~>"
143
143
  - !ruby/object:Gem::Version
144
144
  version: '1.6'
145
145
  name: bundler
@@ -147,13 +147,13 @@ dependencies:
147
147
  type: :development
148
148
  version_requirements: !ruby/object:Gem::Requirement
149
149
  requirements:
150
- - - ~>
150
+ - - "~>"
151
151
  - !ruby/object:Gem::Version
152
152
  version: '1.6'
153
153
  - !ruby/object:Gem::Dependency
154
154
  requirement: !ruby/object:Gem::Requirement
155
155
  requirements:
156
- - - ~>
156
+ - - "~>"
157
157
  - !ruby/object:Gem::Version
158
158
  version: '10.0'
159
159
  name: rake
@@ -161,9 +161,23 @@ dependencies:
161
161
  type: :development
162
162
  version_requirements: !ruby/object:Gem::Requirement
163
163
  requirements:
164
- - - ~>
164
+ - - "~>"
165
165
  - !ruby/object:Gem::Version
166
166
  version: '10.0'
167
+ - !ruby/object:Gem::Dependency
168
+ requirement: !ruby/object:Gem::Requirement
169
+ requirements:
170
+ - - "~>"
171
+ - !ruby/object:Gem::Version
172
+ version: '3.3'
173
+ name: rspec
174
+ prerelease: false
175
+ type: :development
176
+ version_requirements: !ruby/object:Gem::Requirement
177
+ requirements:
178
+ - - "~>"
179
+ - !ruby/object:Gem::Version
180
+ version: '3.3'
167
181
  description: Rbitter archives all tweets appeared on user streaming using ActiveRecord. XMLRPC is used to serve archived tweets and useful features
168
182
  email:
169
183
  - nidev.plontra@gmail.com
@@ -172,9 +186,9 @@ executables:
172
186
  extensions: []
173
187
  extra_rdoc_files: []
174
188
  files:
175
- - .gitignore
176
- - .rspec
177
- - .travis.yml
189
+ - ".gitignore"
190
+ - ".rspec"
191
+ - ".travis.yml"
178
192
  - Gemfile
179
193
  - LICENSE.txt
180
194
  - README.md
@@ -233,17 +247,17 @@ require_paths:
233
247
  - lib
234
248
  required_ruby_version: !ruby/object:Gem::Requirement
235
249
  requirements:
236
- - - '>='
250
+ - - ">="
237
251
  - !ruby/object:Gem::Version
238
252
  version: '0'
239
253
  required_rubygems_version: !ruby/object:Gem::Requirement
240
254
  requirements:
241
- - - '>='
255
+ - - ">="
242
256
  - !ruby/object:Gem::Version
243
257
  version: '0'
244
258
  requirements: []
245
259
  rubyforge_project:
246
- rubygems_version: 2.4.5
260
+ rubygems_version: 2.4.8
247
261
  signing_key:
248
262
  specification_version: 4
249
263
  summary: Rbitter is a Twitter client specialized in archiving
@@ -269,3 +283,4 @@ test_files:
269
283
  - spec/rbitter_spec.rb
270
284
  - spec/sample_data/.keep
271
285
  - spec/spec_helper.rb
286
+ has_rdoc: