rbitter 0.2.1-java → 0.2.3-java

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 06bfae2b2f77d2fc7187c107ddcb42f3b8240860
4
- data.tar.gz: 6956d800f7d306b1ae4b6b1b78b3a4e1c383e954
3
+ metadata.gz: cc1c405538b3bf82c8ee56ab921ac307e95b9b92
4
+ data.tar.gz: 0134c8c6118a264f544509b23a4a113a46eeb511
5
5
  SHA512:
6
- metadata.gz: 8d745a88bc36645e6e3b2b2224a192a44e4ca6dfb09f136b82ba275bdcc29abe2fe01ff7c51bd9dd9d8d1187eaab56b58e66cf8977c4da625b2c8c547036efb9
7
- data.tar.gz: 1676d89e1c4b4fdb7b24a3d0f6d9263148d327fb82547de298dc7ed81f734df502a734bd083b61b3c003e59719b6867cf19e25a46a4be78ab0f5ee667b0962bb
6
+ metadata.gz: f41ac58c4de23d27cc99b6a6a2627a3da1225f69550be1c2571dbe29028d4d95eaf2b7bf07df29320e4fae570c3b6fc461492b1a6b2ef2fc2c282c13597e8fe1
7
+ data.tar.gz: d2bc79071ee6f265abb790de90f5059a7761b32f2e023d40fc6418a20cfe8e46cbb505d7eba3e1511a3f26128e4b8a861727a6e3479f719e963040ccc54af028
data/Gemfile CHANGED
@@ -1,7 +1,6 @@
1
1
  source "https://rubygems.org"
2
2
 
3
3
  group :test do
4
- gem 'rspec', '~>3.0'
5
4
  gem 'simplecov', :require => false, :group => :test
6
5
  gem 'coveralls', :require => false
7
6
  end
@@ -114,6 +114,7 @@ module Rbitter
114
114
 
115
115
  streaming_adapter.new(Rbitter['twitter']).run { |a|
116
116
  @dt << a['media_urls']
117
+ draw "[rbitter] saving @#{a['screen_name']}:#{a['tweetid']}"
117
118
 
118
119
  record = Record.find_or_initialize_by(tweetid: a['tweetid'])
119
120
  record.update({:marker => 0,
@@ -128,7 +129,7 @@ module Rbitter
128
129
  :fav_count => a['fav_count']})
129
130
 
130
131
  record.save
131
- draw "[rbitter] saving tweet: #{a['tweetid']}"
132
+ draw "[rbitter] saved @#{a['screen_name']}:#{a['tweetid']}"
132
133
  }
133
134
  rescue Interrupt => e
134
135
  puts ""
@@ -146,7 +147,7 @@ module Rbitter
146
147
 
147
148
  retry if resurrect_loop?
148
149
  rescue Resolv::ResolvError, Errno::ECONNABORTED,
149
- Errno::ECONNREFUSED, Errno::ECONNRESET => e
150
+ Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ETIMEDOUT => e
150
151
  puts "Network problem. Retry in 5 seconds..."
151
152
  mark_error(e.to_s, "(retry) Network problem")
152
153
 
@@ -5,6 +5,8 @@ require "openssl"
5
5
 
6
6
  module Rbitter
7
7
  class DLThread
8
+ MAX_THREADS = 20
9
+
8
10
  def initialize(dlfolder, large_flag)
9
11
  @dest = dlfolder
10
12
  if not File.directory?(dlfolder)
@@ -23,27 +25,34 @@ module Rbitter
23
25
  end
24
26
 
25
27
  def <<(url_array)
26
- download_task = Thread.new {
28
+ if @pool.length >= MAX_THREADS
29
+ job_cleanup
30
+ end
31
+
32
+ download_task = Thread.new do
27
33
  url_array.each { |url|
28
34
  uri = URI.parse(@large_image ? url + ":large" : url)
29
- ssl = uri.scheme.downcase == 'https'
30
-
31
- Net::HTTP.start(uri.host, uri.port, :use_ssl => ssl) { |h|
32
- req = Net::HTTP::Get.new uri.request_uri
33
- h.request(req) { |res|
34
- case res
35
- when Net::HTTPOK
36
- fname = File.basename(url)
37
-
38
- puts "[fetch] remote: #{uri.path} => local: #{fname}"
39
- open(File.join(@dest, fname), "wb") { |file|
40
- res.read_body { |chunk| file.write(chunk) }
41
- }
42
- end
35
+
36
+ begin
37
+ Net::HTTP.start(uri.host, uri.port, :use_ssl => uri.scheme.downcase == 'https') { |h|
38
+ req = Net::HTTP::Get.new uri.request_uri
39
+ h.request(req) { |res|
40
+ case res
41
+ when Net::HTTPOK
42
+ fname = File.basename(url)
43
+
44
+ puts "[fetch] remote: #{uri.path} => local: #{fname}"
45
+ open(File.join(@dest, fname), "wb") { |file|
46
+ res.read_body { |chunk| file.write(chunk) }
47
+ }
48
+ end
49
+ }
43
50
  }
44
- }
51
+ rescue => e
52
+ puts "[dlthread] exception from thread -> #{e.to_s}"
53
+ end
45
54
  }
46
- }
55
+ end
47
56
 
48
57
  @pool.push download_task
49
58
  end
@@ -52,12 +61,10 @@ module Rbitter
52
61
  until @pool.empty?
53
62
  dlthrd = @pool.shift
54
63
 
55
- if dlthrd.alive?
56
- puts "[dlthread] Thread forceful cleaning up [remains: #{@pool.length}]"
57
- dlthrd.terminate
58
- dlthrd.join
64
+ if dlthrd.join(5).nil?
65
+ puts "[dlthread] #{dlthrd.to_s} is still running. (timeout: 5sec)"
59
66
  end
60
67
  end
61
68
  end
62
69
  end
63
- end
70
+ end
data/lib/rbitter/env.rb CHANGED
@@ -63,7 +63,7 @@ module Rbitter
63
63
  # Level: warning since it is not utilized at all.
64
64
  currents.each { |conf|
65
65
  unless defaults.include?(conf)
66
- warn "[config.json] Unused config: #{conf}. You can safely remove it."
66
+ warn "[config.json] Unsupported field: #{conf}"
67
67
  end
68
68
  }
69
69
 
@@ -71,7 +71,7 @@ module Rbitter
71
71
  # Level: error and program should stop. (return false for this)
72
72
  defaults.each { |conf|
73
73
  unless currents.include?(conf)
74
- warn "[config.json] Config not found: #{conf}. Invalid configuration!"
74
+ warn "[config.json] Error! Missing field: #{conf}"
75
75
  not_errored = false
76
76
  end
77
77
  }
@@ -88,9 +88,9 @@ module Rbitter
88
88
  }
89
89
 
90
90
  return @@env if env_valid?
91
- fail StandardError, "Invalid configuration"
91
+ fail StandardError, "Void configuration. Maybe failure on load?"
92
92
  rescue => e
93
- fail ConfigFileError, "Load Failure (#{json_path}): #{e.to_s}"
93
+ fail ConfigFileError, "Config error on (#{json_path}): #{e.to_s}"
94
94
  end
95
95
  end
96
96
 
@@ -111,6 +111,6 @@ module Rbitter
111
111
  fail ConfigFileError, "No config.json on #{locations.join(' or ')}" if @@env.empty?
112
112
  fail ConfigFileError, "Configuration outdated. Please see above messages to update it" if not env_valid?
113
113
 
114
- puts "[config.json] Loaded configuration is valid. good to go!"
114
+ puts "[config.json] JSON data structure is safe to go."
115
115
  end
116
116
  end
@@ -10,7 +10,10 @@ module Rbitter
10
10
  def draw sentence
11
11
  $> << sentence
12
12
  if sentence.length < @@last_draw
13
- $> << " "*(@@last_draw - sentence.length)
13
+ clear_char_len = @@last_draw - sentence.length
14
+ clear_char_len.times {
15
+ $> << " "
16
+ }
14
17
  end
15
18
  @@last_draw = sentence.length
16
19
  putback
@@ -47,7 +47,10 @@ module ARSupport
47
47
  timeout: 10000) # Long timeout for slow computer
48
48
  end
49
49
  elsif Rbitter['activerecord'] == 'mysql2'
50
- Jdbc::MySQL.load_driver if RUBY_PLATFORM == 'java'
50
+ if RUBY_PLATFORM == 'java'
51
+ require 'jdbc/mysql'
52
+ Jdbc::MySQL.load_driver
53
+ end
51
54
 
52
55
  ActiveRecord::Base.establish_connection(
53
56
  adapter: (RUBY_PLATFORM == 'java' ? 'jdbcmysql' : 'mysql2'),
@@ -1,6 +1,6 @@
1
1
  module Rbitter
2
2
  PRODUCT_NAME = "Rbitter"
3
- VERSION = "0.2.1"
3
+ VERSION = "0.2.3"
4
4
 
5
5
  def major
6
6
  VERSION.match(/^([0-9]+)\./)[1]
data/rbitter.gemspec CHANGED
@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
23
23
  spec.add_dependency 'twitter', '~> 5.14'
24
24
  spec.add_dependency 'json', '~> 1.7'
25
25
  spec.add_dependency 'ripl', '~> 0.7'
26
- spec.add_dependency 'activerecord', '~> 4.0'
26
+ spec.add_dependency 'activerecord', '~> 4.2'
27
27
 
28
28
  if RUBY_PLATFORM == 'java'
29
29
  spec.platform = 'java'
@@ -38,9 +38,9 @@ Gem::Specification.new do |spec|
38
38
 
39
39
  spec.add_dependency 'sqlite3', '~> 1.3'
40
40
  spec.add_dependency 'mysql2', '~> 0.3'
41
- spec.add_dependency 'activerecord-mysql2-adapter', '~> 0.0.3'
42
41
  end
43
42
 
44
43
  spec.add_development_dependency "bundler", "~> 1.6"
45
44
  spec.add_development_dependency "rake", "~> 10.0"
45
+ spec.add_development_dependency "rspec", "~> 3.3"
46
46
  end
metadata CHANGED
@@ -1,19 +1,19 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rbitter
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.1
4
+ version: 0.2.3
5
5
  platform: java
6
6
  authors:
7
7
  - Nidev Plontra
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-06-20 00:00:00.000000000 Z
11
+ date: 2015-08-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
15
15
  requirements:
16
- - - ~>
16
+ - - "~>"
17
17
  - !ruby/object:Gem::Version
18
18
  version: '5.14'
19
19
  name: twitter
@@ -21,13 +21,13 @@ dependencies:
21
21
  type: :runtime
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ~>
24
+ - - "~>"
25
25
  - !ruby/object:Gem::Version
26
26
  version: '5.14'
27
27
  - !ruby/object:Gem::Dependency
28
28
  requirement: !ruby/object:Gem::Requirement
29
29
  requirements:
30
- - - ~>
30
+ - - "~>"
31
31
  - !ruby/object:Gem::Version
32
32
  version: '1.7'
33
33
  name: json
@@ -35,13 +35,13 @@ dependencies:
35
35
  type: :runtime
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - ~>
38
+ - - "~>"
39
39
  - !ruby/object:Gem::Version
40
40
  version: '1.7'
41
41
  - !ruby/object:Gem::Dependency
42
42
  requirement: !ruby/object:Gem::Requirement
43
43
  requirements:
44
- - - ~>
44
+ - - "~>"
45
45
  - !ruby/object:Gem::Version
46
46
  version: '0.7'
47
47
  name: ripl
@@ -49,27 +49,27 @@ dependencies:
49
49
  type: :runtime
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - ~>
52
+ - - "~>"
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0.7'
55
55
  - !ruby/object:Gem::Dependency
56
56
  requirement: !ruby/object:Gem::Requirement
57
57
  requirements:
58
- - - ~>
58
+ - - "~>"
59
59
  - !ruby/object:Gem::Version
60
- version: '4.0'
60
+ version: '4.2'
61
61
  name: activerecord
62
62
  prerelease: false
63
63
  type: :runtime
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - ~>
66
+ - - "~>"
67
67
  - !ruby/object:Gem::Version
68
- version: '4.0'
68
+ version: '4.2'
69
69
  - !ruby/object:Gem::Dependency
70
70
  requirement: !ruby/object:Gem::Requirement
71
71
  requirements:
72
- - - ~>
72
+ - - "~>"
73
73
  - !ruby/object:Gem::Version
74
74
  version: '1.3'
75
75
  name: activerecord-jdbc-adapter
@@ -77,13 +77,13 @@ dependencies:
77
77
  type: :runtime
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
- - - ~>
80
+ - - "~>"
81
81
  - !ruby/object:Gem::Version
82
82
  version: '1.3'
83
83
  - !ruby/object:Gem::Dependency
84
84
  requirement: !ruby/object:Gem::Requirement
85
85
  requirements:
86
- - - ~>
86
+ - - "~>"
87
87
  - !ruby/object:Gem::Version
88
88
  version: '3.8'
89
89
  name: jdbc-sqlite3
@@ -91,13 +91,13 @@ dependencies:
91
91
  type: :runtime
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
- - - ~>
94
+ - - "~>"
95
95
  - !ruby/object:Gem::Version
96
96
  version: '3.8'
97
97
  - !ruby/object:Gem::Dependency
98
98
  requirement: !ruby/object:Gem::Requirement
99
99
  requirements:
100
- - - ~>
100
+ - - "~>"
101
101
  - !ruby/object:Gem::Version
102
102
  version: '5.1'
103
103
  name: jdbc-mysql
@@ -105,13 +105,13 @@ dependencies:
105
105
  type: :runtime
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
- - - ~>
108
+ - - "~>"
109
109
  - !ruby/object:Gem::Version
110
110
  version: '5.1'
111
111
  - !ruby/object:Gem::Dependency
112
112
  requirement: !ruby/object:Gem::Requirement
113
113
  requirements:
114
- - - ~>
114
+ - - "~>"
115
115
  - !ruby/object:Gem::Version
116
116
  version: '1.3'
117
117
  name: activerecord-jdbcsqlite3-adapter
@@ -119,13 +119,13 @@ dependencies:
119
119
  type: :runtime
120
120
  version_requirements: !ruby/object:Gem::Requirement
121
121
  requirements:
122
- - - ~>
122
+ - - "~>"
123
123
  - !ruby/object:Gem::Version
124
124
  version: '1.3'
125
125
  - !ruby/object:Gem::Dependency
126
126
  requirement: !ruby/object:Gem::Requirement
127
127
  requirements:
128
- - - ~>
128
+ - - "~>"
129
129
  - !ruby/object:Gem::Version
130
130
  version: '1.3'
131
131
  name: activerecord-jdbcmysql-adapter
@@ -133,13 +133,13 @@ dependencies:
133
133
  type: :runtime
134
134
  version_requirements: !ruby/object:Gem::Requirement
135
135
  requirements:
136
- - - ~>
136
+ - - "~>"
137
137
  - !ruby/object:Gem::Version
138
138
  version: '1.3'
139
139
  - !ruby/object:Gem::Dependency
140
140
  requirement: !ruby/object:Gem::Requirement
141
141
  requirements:
142
- - - ~>
142
+ - - "~>"
143
143
  - !ruby/object:Gem::Version
144
144
  version: '1.6'
145
145
  name: bundler
@@ -147,13 +147,13 @@ dependencies:
147
147
  type: :development
148
148
  version_requirements: !ruby/object:Gem::Requirement
149
149
  requirements:
150
- - - ~>
150
+ - - "~>"
151
151
  - !ruby/object:Gem::Version
152
152
  version: '1.6'
153
153
  - !ruby/object:Gem::Dependency
154
154
  requirement: !ruby/object:Gem::Requirement
155
155
  requirements:
156
- - - ~>
156
+ - - "~>"
157
157
  - !ruby/object:Gem::Version
158
158
  version: '10.0'
159
159
  name: rake
@@ -161,9 +161,23 @@ dependencies:
161
161
  type: :development
162
162
  version_requirements: !ruby/object:Gem::Requirement
163
163
  requirements:
164
- - - ~>
164
+ - - "~>"
165
165
  - !ruby/object:Gem::Version
166
166
  version: '10.0'
167
+ - !ruby/object:Gem::Dependency
168
+ requirement: !ruby/object:Gem::Requirement
169
+ requirements:
170
+ - - "~>"
171
+ - !ruby/object:Gem::Version
172
+ version: '3.3'
173
+ name: rspec
174
+ prerelease: false
175
+ type: :development
176
+ version_requirements: !ruby/object:Gem::Requirement
177
+ requirements:
178
+ - - "~>"
179
+ - !ruby/object:Gem::Version
180
+ version: '3.3'
167
181
  description: Rbitter archives all tweets appeared on user streaming using ActiveRecord. XMLRPC is used to serve archived tweets and useful features
168
182
  email:
169
183
  - nidev.plontra@gmail.com
@@ -172,9 +186,9 @@ executables:
172
186
  extensions: []
173
187
  extra_rdoc_files: []
174
188
  files:
175
- - .gitignore
176
- - .rspec
177
- - .travis.yml
189
+ - ".gitignore"
190
+ - ".rspec"
191
+ - ".travis.yml"
178
192
  - Gemfile
179
193
  - LICENSE.txt
180
194
  - README.md
@@ -233,17 +247,17 @@ require_paths:
233
247
  - lib
234
248
  required_ruby_version: !ruby/object:Gem::Requirement
235
249
  requirements:
236
- - - '>='
250
+ - - ">="
237
251
  - !ruby/object:Gem::Version
238
252
  version: '0'
239
253
  required_rubygems_version: !ruby/object:Gem::Requirement
240
254
  requirements:
241
- - - '>='
255
+ - - ">="
242
256
  - !ruby/object:Gem::Version
243
257
  version: '0'
244
258
  requirements: []
245
259
  rubyforge_project:
246
- rubygems_version: 2.4.5
260
+ rubygems_version: 2.4.8
247
261
  signing_key:
248
262
  specification_version: 4
249
263
  summary: Rbitter is a Twitter client specialized in archiving
@@ -269,3 +283,4 @@ test_files:
269
283
  - spec/rbitter_spec.rb
270
284
  - spec/sample_data/.keep
271
285
  - spec/spec_helper.rb
286
+ has_rdoc: