pluto-models 1.5.2 → 1.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +1 -1
- data/lib/pluto/models/feed.rb +64 -62
- data/lib/pluto/version.rb +1 -1
- metadata +4 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 92f609c55f1c97dce198ee7f9a8513d1937efc92
|
|
4
|
+
data.tar.gz: b688795ca55c96259a5ef2494fb3fe24fd63fb65
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e603bb5436e331469dc36edea780f52e831ccce781e43f79bcaf7c83fdb57f1599271a889b6b670b18ddc0542d2e69901c2f7f5526c11ba1b062f525d1096b5f
|
|
7
|
+
data.tar.gz: 62499df65996003274e377b6a9c9c5c8e2f398c1dbac7601b9daf4d3201295f89c13d4b8ce9f0641d8123e292444801d2d6a56bce45e25c8d0e7db66200ae997
|
data/Rakefile
CHANGED
data/lib/pluto/models/feed.rb
CHANGED
|
@@ -89,85 +89,87 @@ class Feed < ActiveRecord::Base
|
|
|
89
89
|
|
|
90
90
|
logger = LogUtils::Logger.root
|
|
91
91
|
|
|
92
|
-
|
|
93
|
-
##
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
data.items.each do |item|
|
|
103
|
-
if includesFilter && includesFilter.match_item?( item ) == false
|
|
104
|
-
logger.info "** SKIPPING | #{item.title}"
|
|
105
|
-
logger.info " no include terms match: #{includes}"
|
|
106
|
-
next ## skip to next item
|
|
107
|
-
end
|
|
108
|
-
|
|
109
|
-
item_rec = Item.find_by_guid( item.guid )
|
|
110
|
-
if item_rec.nil?
|
|
111
|
-
item_rec = Item.new
|
|
112
|
-
logger.info "** NEW | #{item.title}"
|
|
92
|
+
## note: handle case with empty feed, that is, feed with NO items / entries
|
|
93
|
+
## (e.g. data.items.size == 0).
|
|
94
|
+
if data.items.size > 0
|
|
95
|
+
|
|
96
|
+
######
|
|
97
|
+
## check for filters (includes/excludes) if present
|
|
98
|
+
## for now just check for includes
|
|
99
|
+
##
|
|
100
|
+
if includes.present?
|
|
101
|
+
includesFilter = FeedFilter::IncludeFilters.new( includes )
|
|
113
102
|
else
|
|
114
|
-
|
|
115
|
-
logger.info "UPDATE | #{item.title}"
|
|
103
|
+
includesFilter = nil
|
|
116
104
|
end
|
|
117
105
|
|
|
118
|
-
|
|
106
|
+
data.items.each do |item|
|
|
107
|
+
if includesFilter && includesFilter.match_item?( item ) == false
|
|
108
|
+
logger.info "** SKIPPING | #{item.title}"
|
|
109
|
+
logger.info " no include terms match: #{includes}"
|
|
110
|
+
next ## skip to next item
|
|
111
|
+
end
|
|
119
112
|
|
|
120
|
-
|
|
121
|
-
|
|
113
|
+
item_rec = Item.find_by_guid( item.guid )
|
|
114
|
+
if item_rec.nil?
|
|
115
|
+
item_rec = Item.new
|
|
116
|
+
logger.info "** NEW | #{item.title}"
|
|
117
|
+
else
|
|
118
|
+
## todo: check if any attribs changed
|
|
119
|
+
logger.info "UPDATE | #{item.title}"
|
|
120
|
+
end
|
|
122
121
|
|
|
123
|
-
|
|
122
|
+
item_rec.debug = debug? ? true : false # pass along debug flag
|
|
124
123
|
|
|
125
|
-
|
|
124
|
+
item_rec.feed_id = id # feed_rec.id - add feed_id fk_ref
|
|
125
|
+
item_rec.fetched = fetched # feed_rec.fetched
|
|
126
126
|
|
|
127
|
+
item_rec.update_from_struct!( item )
|
|
128
|
+
end # each item
|
|
127
129
|
|
|
128
|
-
###
|
|
129
|
-
# delete (old) feed items if no longer in feed AND
|
|
130
|
-
# date range is in (lastest/current) feed list
|
|
131
|
-
#
|
|
132
|
-
# thanks to Harry Wood
|
|
133
|
-
# see https://github.com/feedreader/pluto/pull/16
|
|
134
|
-
# for more comments
|
|
135
130
|
|
|
136
|
-
|
|
137
|
-
|
|
131
|
+
###
|
|
132
|
+
# delete (old) feed items if no longer in feed AND
|
|
133
|
+
# date range is in (lastest/current) feed list
|
|
134
|
+
#
|
|
135
|
+
# thanks to Harry Wood
|
|
136
|
+
# see https://github.com/feedreader/pluto/pull/16
|
|
137
|
+
# for more comments
|
|
138
138
|
|
|
139
|
+
# todo/fix: use a delete feature/command line flag to make it optional - why? why not?
|
|
139
140
|
|
|
140
|
-
|
|
141
|
-
|
|
141
|
+
guids_in_feed = data.items.map {|item| item.guid }
|
|
142
|
+
earliest_still_in_feed = data.items.min_by {|item| item.published }.published
|
|
142
143
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
144
|
+
items_no_longer_present =
|
|
145
|
+
Item
|
|
146
|
+
.where(feed_id: id)
|
|
147
|
+
.where.not(published: nil)
|
|
148
|
+
.where("published > ?", earliest_still_in_feed)
|
|
149
|
+
.where.not(guid: guids_in_feed)
|
|
149
150
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
151
|
+
unless items_no_longer_present.empty?
|
|
152
|
+
logger.info "#{items_no_longer_present.size} items no longer present in the feed (presumed removed at source). Deleting from planet db"
|
|
153
|
+
items_no_longer_present.each do |item|
|
|
154
|
+
logger.info "** DELETE | #{item.title}"
|
|
155
|
+
item.destroy
|
|
156
|
+
end
|
|
155
157
|
end
|
|
156
|
-
end
|
|
157
158
|
|
|
158
159
|
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
160
|
+
# update cached value last published for item
|
|
161
|
+
## todo/check: force reload of items - why? why not??
|
|
162
|
+
last_item_rec = items.latest.limit(1).first # note limit(1) will return relation/arrar - use first to get first element or nil from ary
|
|
163
|
+
if last_item_rec.present?
|
|
164
|
+
if last_item_rec.updated?
|
|
165
|
+
self.items_last_updated = last_item_rec.updated
|
|
166
|
+
## save! ## note: will get save w/ update_from_struct! - why? why not??
|
|
167
|
+
else # try published
|
|
168
|
+
self.items_last_updated = last_item_rec.published
|
|
169
|
+
## save! ## note: will get save w/ update_from_struct! - why? why not??
|
|
170
|
+
end
|
|
169
171
|
end
|
|
170
|
-
end
|
|
172
|
+
end # check for if data.items.size > 0 (that is, feed has feed items/entries)
|
|
171
173
|
|
|
172
174
|
update_from_struct!( data )
|
|
173
175
|
end # method deep_update_from_struct!
|
data/lib/pluto/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: pluto-models
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.5.
|
|
4
|
+
version: 1.5.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Gerald Bauer
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2020-01-
|
|
11
|
+
date: 2020-01-30 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: props
|
|
@@ -44,14 +44,14 @@ dependencies:
|
|
|
44
44
|
requirements:
|
|
45
45
|
- - ">="
|
|
46
46
|
- !ruby/object:Gem::Version
|
|
47
|
-
version: 2.1.
|
|
47
|
+
version: 2.1.2
|
|
48
48
|
type: :runtime
|
|
49
49
|
prerelease: false
|
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
|
51
51
|
requirements:
|
|
52
52
|
- - ">="
|
|
53
53
|
- !ruby/object:Gem::Version
|
|
54
|
-
version: 2.1.
|
|
54
|
+
version: 2.1.2
|
|
55
55
|
- !ruby/object:Gem::Dependency
|
|
56
56
|
name: feedfilter
|
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|