pluto-models 1.5.2 → 1.5.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Rakefile +1 -1
- data/lib/pluto/models/feed.rb +64 -62
- data/lib/pluto/version.rb +1 -1
- metadata +4 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 92f609c55f1c97dce198ee7f9a8513d1937efc92
|
|
4
|
+
data.tar.gz: b688795ca55c96259a5ef2494fb3fe24fd63fb65
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e603bb5436e331469dc36edea780f52e831ccce781e43f79bcaf7c83fdb57f1599271a889b6b670b18ddc0542d2e69901c2f7f5526c11ba1b062f525d1096b5f
|
|
7
|
+
data.tar.gz: 62499df65996003274e377b6a9c9c5c8e2f398c1dbac7601b9daf4d3201295f89c13d4b8ce9f0641d8123e292444801d2d6a56bce45e25c8d0e7db66200ae997
|
data/Rakefile
CHANGED
data/lib/pluto/models/feed.rb
CHANGED
|
@@ -89,85 +89,87 @@ class Feed < ActiveRecord::Base
|
|
|
89
89
|
|
|
90
90
|
logger = LogUtils::Logger.root
|
|
91
91
|
|
|
92
|
-
|
|
93
|
-
##
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
data.items.each do |item|
|
|
103
|
-
if includesFilter && includesFilter.match_item?( item ) == false
|
|
104
|
-
logger.info "** SKIPPING | #{item.title}"
|
|
105
|
-
logger.info " no include terms match: #{includes}"
|
|
106
|
-
next ## skip to next item
|
|
107
|
-
end
|
|
108
|
-
|
|
109
|
-
item_rec = Item.find_by_guid( item.guid )
|
|
110
|
-
if item_rec.nil?
|
|
111
|
-
item_rec = Item.new
|
|
112
|
-
logger.info "** NEW | #{item.title}"
|
|
92
|
+
## note: handle case with empty feed, that is, feed with NO items / entries
|
|
93
|
+
## (e.g. data.items.size == 0).
|
|
94
|
+
if data.items.size > 0
|
|
95
|
+
|
|
96
|
+
######
|
|
97
|
+
## check for filters (includes/excludes) if present
|
|
98
|
+
## for now just check for includes
|
|
99
|
+
##
|
|
100
|
+
if includes.present?
|
|
101
|
+
includesFilter = FeedFilter::IncludeFilters.new( includes )
|
|
113
102
|
else
|
|
114
|
-
|
|
115
|
-
logger.info "UPDATE | #{item.title}"
|
|
103
|
+
includesFilter = nil
|
|
116
104
|
end
|
|
117
105
|
|
|
118
|
-
|
|
106
|
+
data.items.each do |item|
|
|
107
|
+
if includesFilter && includesFilter.match_item?( item ) == false
|
|
108
|
+
logger.info "** SKIPPING | #{item.title}"
|
|
109
|
+
logger.info " no include terms match: #{includes}"
|
|
110
|
+
next ## skip to next item
|
|
111
|
+
end
|
|
119
112
|
|
|
120
|
-
|
|
121
|
-
|
|
113
|
+
item_rec = Item.find_by_guid( item.guid )
|
|
114
|
+
if item_rec.nil?
|
|
115
|
+
item_rec = Item.new
|
|
116
|
+
logger.info "** NEW | #{item.title}"
|
|
117
|
+
else
|
|
118
|
+
## todo: check if any attribs changed
|
|
119
|
+
logger.info "UPDATE | #{item.title}"
|
|
120
|
+
end
|
|
122
121
|
|
|
123
|
-
|
|
122
|
+
item_rec.debug = debug? ? true : false # pass along debug flag
|
|
124
123
|
|
|
125
|
-
|
|
124
|
+
item_rec.feed_id = id # feed_rec.id - add feed_id fk_ref
|
|
125
|
+
item_rec.fetched = fetched # feed_rec.fetched
|
|
126
126
|
|
|
127
|
+
item_rec.update_from_struct!( item )
|
|
128
|
+
end # each item
|
|
127
129
|
|
|
128
|
-
###
|
|
129
|
-
# delete (old) feed items if no longer in feed AND
|
|
130
|
-
# date range is in (lastest/current) feed list
|
|
131
|
-
#
|
|
132
|
-
# thanks to Harry Wood
|
|
133
|
-
# see https://github.com/feedreader/pluto/pull/16
|
|
134
|
-
# for more comments
|
|
135
130
|
|
|
136
|
-
|
|
137
|
-
|
|
131
|
+
###
|
|
132
|
+
# delete (old) feed items if no longer in feed AND
|
|
133
|
+
# date range is in (lastest/current) feed list
|
|
134
|
+
#
|
|
135
|
+
# thanks to Harry Wood
|
|
136
|
+
# see https://github.com/feedreader/pluto/pull/16
|
|
137
|
+
# for more comments
|
|
138
138
|
|
|
139
|
+
# todo/fix: use a delete feature/command line flag to make it optional - why? why not?
|
|
139
140
|
|
|
140
|
-
|
|
141
|
-
|
|
141
|
+
guids_in_feed = data.items.map {|item| item.guid }
|
|
142
|
+
earliest_still_in_feed = data.items.min_by {|item| item.published }.published
|
|
142
143
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
144
|
+
items_no_longer_present =
|
|
145
|
+
Item
|
|
146
|
+
.where(feed_id: id)
|
|
147
|
+
.where.not(published: nil)
|
|
148
|
+
.where("published > ?", earliest_still_in_feed)
|
|
149
|
+
.where.not(guid: guids_in_feed)
|
|
149
150
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
151
|
+
unless items_no_longer_present.empty?
|
|
152
|
+
logger.info "#{items_no_longer_present.size} items no longer present in the feed (presumed removed at source). Deleting from planet db"
|
|
153
|
+
items_no_longer_present.each do |item|
|
|
154
|
+
logger.info "** DELETE | #{item.title}"
|
|
155
|
+
item.destroy
|
|
156
|
+
end
|
|
155
157
|
end
|
|
156
|
-
end
|
|
157
158
|
|
|
158
159
|
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
160
|
+
# update cached value last published for item
|
|
161
|
+
## todo/check: force reload of items - why? why not??
|
|
162
|
+
last_item_rec = items.latest.limit(1).first # note limit(1) will return relation/arrar - use first to get first element or nil from ary
|
|
163
|
+
if last_item_rec.present?
|
|
164
|
+
if last_item_rec.updated?
|
|
165
|
+
self.items_last_updated = last_item_rec.updated
|
|
166
|
+
## save! ## note: will get save w/ update_from_struct! - why? why not??
|
|
167
|
+
else # try published
|
|
168
|
+
self.items_last_updated = last_item_rec.published
|
|
169
|
+
## save! ## note: will get save w/ update_from_struct! - why? why not??
|
|
170
|
+
end
|
|
169
171
|
end
|
|
170
|
-
end
|
|
172
|
+
end # check for if data.items.size > 0 (that is, feed has feed items/entries)
|
|
171
173
|
|
|
172
174
|
update_from_struct!( data )
|
|
173
175
|
end # method deep_update_from_struct!
|
data/lib/pluto/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: pluto-models
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.5.
|
|
4
|
+
version: 1.5.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Gerald Bauer
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2020-01-
|
|
11
|
+
date: 2020-01-30 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: props
|
|
@@ -44,14 +44,14 @@ dependencies:
|
|
|
44
44
|
requirements:
|
|
45
45
|
- - ">="
|
|
46
46
|
- !ruby/object:Gem::Version
|
|
47
|
-
version: 2.1.
|
|
47
|
+
version: 2.1.2
|
|
48
48
|
type: :runtime
|
|
49
49
|
prerelease: false
|
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
|
51
51
|
requirements:
|
|
52
52
|
- - ">="
|
|
53
53
|
- !ruby/object:Gem::Version
|
|
54
|
-
version: 2.1.
|
|
54
|
+
version: 2.1.2
|
|
55
55
|
- !ruby/object:Gem::Dependency
|
|
56
56
|
name: feedfilter
|
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|