pluto-update 1.6.2 → 1.6.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Manifest.txt +0 -1
- data/Rakefile +2 -4
- data/lib/pluto/update.rb +3 -13
- data/lib/pluto/update/feed_refresher.rb +4 -18
- data/lib/pluto/update/site_fetcher.rb +6 -40
- data/lib/pluto/update/site_refresher.rb +3 -9
- data/lib/pluto/update/version.rb +1 -1
- data/test/helper.rb +4 -4
- data/test/test_refresh.rb +4 -5
- metadata +6 -35
- data/test/data/ruby.ini +0 -13
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: da4ad0bcd2b1b1ce0288b8be1c0ebf5843a28cb0
|
4
|
+
data.tar.gz: 3c70a3e21a8b11411a579635405f61574d78df86
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 687fc3da86da794584d50a904e4ad64ba3db12d21a721fa7f9cb20cff69c6943d858c30c6b05a1f0659ae3f71ad130be4a9b778e633692e8ece89a7e341bdffb
|
7
|
+
data.tar.gz: 3b1281f1372dcfd3ed9e360e012d32f310985796975ca255b2fc25fd5f3061135289c927cd433670f746647e9a2cb5b9ce76b52b05f93396b423e6015275451d
|
data/Manifest.txt
CHANGED
data/Rakefile
CHANGED
@@ -18,10 +18,8 @@ Hoe.spec 'pluto-update' do
|
|
18
18
|
self.history_file = 'CHANGELOG.md'
|
19
19
|
|
20
20
|
self.extra_deps = [
|
21
|
-
['pluto-models', '>= 1.5.
|
22
|
-
['pluto-feedfetcher', '>= 0.1.
|
23
|
-
['fetcher', '>= 0.4.5'],
|
24
|
-
['preproc', '>= 0.1.0'],
|
21
|
+
['pluto-models', '>= 1.5.4'],
|
22
|
+
['pluto-feedfetcher', '>= 0.1.4'],
|
25
23
|
]
|
26
24
|
|
27
25
|
self.licenses = ['Public Domain']
|
data/lib/pluto/update.rb
CHANGED
@@ -5,11 +5,6 @@ require 'pluto/models'
|
|
5
5
|
require 'pluto/feedfetcher'
|
6
6
|
|
7
7
|
|
8
|
-
# more 3rd party gems
|
9
|
-
require 'fetcher' # fetch (text) documents
|
10
|
-
require 'preproc' # include preprocessor
|
11
|
-
|
12
|
-
|
13
8
|
# our own code
|
14
9
|
require 'pluto/update/version' # Note: let version always go first
|
15
10
|
require 'pluto/update/feed_refresher'
|
@@ -28,14 +23,9 @@ module Pluto
|
|
28
23
|
SiteRefresher.new.refresh_sites
|
29
24
|
end
|
30
25
|
|
31
|
-
### convenience
|
32
|
-
def self.update_feeds
|
33
|
-
|
34
|
-
end
|
35
|
-
|
36
|
-
def self.update_sites
|
37
|
-
SiteRefresher.new.refresh_sites
|
38
|
-
end
|
26
|
+
### convenience aliases w/ update_ -- use refresh (only) - why? why not??
|
27
|
+
def self.update_feeds() refresh_feeds; end
|
28
|
+
def self.update_sites() refresh_sites; end
|
39
29
|
|
40
30
|
end # module Pluto
|
41
31
|
|
@@ -18,17 +18,12 @@ class FeedRefresher
|
|
18
18
|
@worker = FeedFetcherCondGetWithCache.new
|
19
19
|
end
|
20
20
|
|
21
|
-
def debug
|
22
|
-
def debug?() @debug || false; end
|
21
|
+
def debug?() Pluto.config.debug?; end
|
23
22
|
|
24
23
|
|
25
|
-
def refresh_feeds( opts={} ) # refresh (fetch+parse+update) all feeds
|
26
|
-
if debug?
|
27
|
-
## turn on logging for sql too
|
28
|
-
ActiveRecord::Base.logger = Logger.new( STDOUT )
|
29
|
-
@worker.debug = true # also pass along worker debug flag if set
|
30
|
-
end
|
31
24
|
|
25
|
+
def refresh_feeds( opts={} ) # refresh (fetch+parse+update) all feeds
|
26
|
+
|
32
27
|
start_time = Time.now
|
33
28
|
Activity.create!( text: "start update feeds (#{Feed.count})" )
|
34
29
|
|
@@ -44,12 +39,7 @@ class FeedRefresher
|
|
44
39
|
|
45
40
|
|
46
41
|
def refresh_feeds_for( site_key, opts={} ) # refresh (fetch+parse+update) feeds for site
|
47
|
-
|
48
|
-
## turn on logging for sql too
|
49
|
-
ActiveRecord::Base.logger = Logger.new( STDOUT )
|
50
|
-
@worker.debug = true # also pass along worker debug flag if set
|
51
|
-
end
|
52
|
-
|
42
|
+
|
53
43
|
# -- log update activity
|
54
44
|
Activity.create!( text: "update feeds >#{site_key}<" )
|
55
45
|
|
@@ -107,10 +97,6 @@ private
|
|
107
97
|
# move to_datetime to feedutils!! if it works
|
108
98
|
## todo: move this comments to feedutils??
|
109
99
|
|
110
|
-
|
111
|
-
feed_rec.debug = debug? ? true : false # pass along debug flag
|
112
|
-
|
113
|
-
## fix/todo: pass debug flag as opts - debug: true|false !!!!!!
|
114
100
|
# fix/todo: find a better name - why? why not?? => use update_from_struct!
|
115
101
|
feed_rec.deep_update_from_struct!( feed )
|
116
102
|
|
@@ -12,8 +12,8 @@ class SiteFetcher
|
|
12
12
|
@worker = Fetcher::Worker.new
|
13
13
|
end
|
14
14
|
|
15
|
-
def debug
|
16
|
-
|
15
|
+
def debug?() Pluto.config.debug?; end
|
16
|
+
|
17
17
|
|
18
18
|
def fetch( site_rec )
|
19
19
|
####################################################
|
@@ -45,18 +45,10 @@ class SiteFetcher
|
|
45
45
|
|
46
46
|
if response.code == '304' # not modified (conditional GET - e.g. using etag/last-modified)
|
47
47
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
##
|
53
|
-
## fix/todo: use ahead-of-time preprocessor ?? in the future to simplify???
|
54
|
-
else
|
55
|
-
puts "OK - fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
|
56
|
-
puts "no change; request returns not modified (304); skipping parsing site config"
|
57
|
-
return nil # no updates available; nothing to do
|
58
|
-
end
|
59
|
-
|
48
|
+
puts "OK - fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
|
49
|
+
puts "no change; request returns not modified (304); skipping parsing site config"
|
50
|
+
return nil # no updates available; nothing to do
|
51
|
+
|
60
52
|
elsif response.code != '200' # note Net::HTTP response.code is a string in ruby
|
61
53
|
|
62
54
|
puts "*** error: fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
|
@@ -97,36 +89,10 @@ class SiteFetcher
|
|
97
89
|
site_rec.update!( site_attribs )
|
98
90
|
|
99
91
|
|
100
|
-
#################
|
101
|
-
### fix: add support for http_etag cache etc. - how??
|
102
|
-
###
|
103
|
-
### use from_text( text, base: base ) !!!!!!!!
|
104
|
-
### do NOT reissue first request
|
105
|
-
##
|
106
|
-
## fix: use special case/method for update_with_includes!!!
|
107
|
-
## keep it simple w/o includes (do NOT mix in one method)
|
108
|
-
## split into two methods!!!
|
109
|
-
|
110
|
-
## retry w/ preprocesser
|
111
|
-
## refetch if @include found w/ all includes included
|
112
|
-
if site_text.index('@include')
|
113
|
-
site_text = InclPreproc.from_url( site_url ).read
|
114
|
-
end
|
115
|
-
|
116
92
|
## logger.debug "site_text:"
|
117
93
|
## logger.debug site_text[ 0..300 ] # get first 300 chars
|
118
94
|
|
119
95
|
site_text
|
120
|
-
|
121
|
-
###
|
122
|
-
## todo/fix:
|
123
|
-
### move INI.load out of this method!! - return site_text or nil
|
124
|
-
##
|
125
|
-
## puts "Before parsing site config >#{site_key}<..."
|
126
|
-
##
|
127
|
-
# assume ini format for now
|
128
|
-
## site_config = INI.load( site_text )
|
129
|
-
## site_config
|
130
96
|
end
|
131
97
|
|
132
98
|
end # class SiteFetcher
|
@@ -18,16 +18,11 @@ class SiteRefresher
|
|
18
18
|
@worker = SiteFetcher.new
|
19
19
|
end
|
20
20
|
|
21
|
-
def debug
|
22
|
-
def debug?() @debug || false; end
|
21
|
+
def debug?() Pluto.config.debug?; end
|
23
22
|
|
24
|
-
def refresh_sites( opts={} ) # refresh (fetch+parse+update) all site configs
|
25
|
-
if debug?
|
26
|
-
## turn on logging for sql too
|
27
|
-
ActiveRecord::Base.logger = Logger.new( STDOUT )
|
28
|
-
@worker.debug = true # also pass along worker debug flag if set
|
29
|
-
end
|
30
23
|
|
24
|
+
def refresh_sites( opts={} ) # refresh (fetch+parse+update) all site configs
|
25
|
+
|
31
26
|
start_time = Time.now
|
32
27
|
Activity.create!( text: "start update sites (#{Site.count})" )
|
33
28
|
|
@@ -50,7 +45,6 @@ private
|
|
50
45
|
|
51
46
|
site_config = INI.load( site_text )
|
52
47
|
|
53
|
-
### site_rec.debug = debug? ? true : false # pass along debug flag
|
54
48
|
site_rec.deep_update_from_hash!( site_config )
|
55
49
|
end
|
56
50
|
|
data/lib/pluto/update/version.rb
CHANGED
data/test/helper.rb
CHANGED
@@ -2,19 +2,19 @@
|
|
2
2
|
|
3
3
|
|
4
4
|
## minitest setup
|
5
|
-
|
6
5
|
require 'minitest/autorun'
|
7
6
|
|
8
7
|
## our own code
|
9
8
|
require 'pluto/update'
|
10
9
|
|
11
10
|
|
12
|
-
|
11
|
+
Pluto.config.debug = true
|
12
|
+
Pluto.config.logger.level = :debug
|
13
13
|
|
14
14
|
|
15
15
|
## some shortcuts
|
16
|
-
Log
|
17
|
-
Prop
|
16
|
+
Log = LogDb::Model::Log
|
17
|
+
Prop = ConfDb::Model::Prop
|
18
18
|
|
19
19
|
Site = Pluto::Model::Site
|
20
20
|
Feed = Pluto::Model::Feed
|
data/test/test_refresh.rb
CHANGED
@@ -1,10 +1,7 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
###
|
4
2
|
# to run use
|
5
3
|
# ruby -I ./lib -I ./test test/test_refresh.rb
|
6
|
-
|
7
|
-
# rake test
|
4
|
+
|
8
5
|
|
9
6
|
require 'helper'
|
10
7
|
|
@@ -16,7 +13,7 @@ class TestRefresh < MiniTest::Test
|
|
16
13
|
Item.delete_all
|
17
14
|
Subscription.delete_all
|
18
15
|
|
19
|
-
site_text = File.read( "#{
|
16
|
+
site_text = File.read( "#{Pluto::Test.data_dir}/ruby.ini")
|
20
17
|
site_config = INI.load( site_text )
|
21
18
|
|
22
19
|
Site.deep_create_or_update_from_hash!( 'ruby', site_config )
|
@@ -25,11 +22,13 @@ class TestRefresh < MiniTest::Test
|
|
25
22
|
|
26
23
|
def test_refresh_sites
|
27
24
|
Pluto.refresh_sites
|
25
|
+
|
28
26
|
assert true
|
29
27
|
end
|
30
28
|
|
31
29
|
def test_refresh_feeds
|
32
30
|
Pluto.refresh_feeds
|
31
|
+
|
33
32
|
assert true
|
34
33
|
end
|
35
34
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pluto-update
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.6.
|
4
|
+
version: 1.6.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Gerald Bauer
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-01
|
11
|
+
date: 2020-02-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: pluto-models
|
@@ -16,56 +16,28 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 1.5.
|
19
|
+
version: 1.5.4
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 1.5.
|
26
|
+
version: 1.5.4
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: pluto-feedfetcher
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: 0.1.
|
33
|
+
version: 0.1.4
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: 0.1.
|
41
|
-
- !ruby/object:Gem::Dependency
|
42
|
-
name: fetcher
|
43
|
-
requirement: !ruby/object:Gem::Requirement
|
44
|
-
requirements:
|
45
|
-
- - ">="
|
46
|
-
- !ruby/object:Gem::Version
|
47
|
-
version: 0.4.5
|
48
|
-
type: :runtime
|
49
|
-
prerelease: false
|
50
|
-
version_requirements: !ruby/object:Gem::Requirement
|
51
|
-
requirements:
|
52
|
-
- - ">="
|
53
|
-
- !ruby/object:Gem::Version
|
54
|
-
version: 0.4.5
|
55
|
-
- !ruby/object:Gem::Dependency
|
56
|
-
name: preproc
|
57
|
-
requirement: !ruby/object:Gem::Requirement
|
58
|
-
requirements:
|
59
|
-
- - ">="
|
60
|
-
- !ruby/object:Gem::Version
|
61
|
-
version: 0.1.0
|
62
|
-
type: :runtime
|
63
|
-
prerelease: false
|
64
|
-
version_requirements: !ruby/object:Gem::Requirement
|
65
|
-
requirements:
|
66
|
-
- - ">="
|
67
|
-
- !ruby/object:Gem::Version
|
68
|
-
version: 0.1.0
|
40
|
+
version: 0.1.4
|
69
41
|
- !ruby/object:Gem::Dependency
|
70
42
|
name: rdoc
|
71
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -112,7 +84,6 @@ files:
|
|
112
84
|
- lib/pluto/update/site_fetcher.rb
|
113
85
|
- lib/pluto/update/site_refresher.rb
|
114
86
|
- lib/pluto/update/version.rb
|
115
|
-
- test/data/ruby.ini
|
116
87
|
- test/helper.rb
|
117
88
|
- test/test_refresh.rb
|
118
89
|
homepage: https://github.com/feedreader/pluto
|
data/test/data/ruby.ini
DELETED
@@ -1,13 +0,0 @@
|
|
1
|
-
title = Planet Ruby
|
2
|
-
source = https://github.com/feedreader/pluto/raw/master/pluto-update/test/data/ruby.ini
|
3
|
-
|
4
|
-
[rubylang]
|
5
|
-
title = Ruby Lang News
|
6
|
-
link = http://www.ruby-lang.org/en/news
|
7
|
-
feed = http://www.ruby-lang.org/en/feeds/news.rss
|
8
|
-
|
9
|
-
[rubyonrails]
|
10
|
-
title = Ruby on Rails News
|
11
|
-
link = http://weblog.rubyonrails.org
|
12
|
-
feed = http://weblog.rubyonrails.org/feed/atom.xml
|
13
|
-
|