polyrex-feed-reader 0.3.3 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data.tar.gz.sig +0 -0
- data/lib/feeds.xsl +48 -0
- data/lib/polyrex-feed-reader.rb +54 -162
- metadata +15 -13
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d2364e173da28ab14324d98fccf4b354425c0ff0
|
4
|
+
data.tar.gz: 461a2e062a34e5129edfd6af447c5f804f5ea3fa
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cce648e7b9d68215de24b57f115de5b8df4b380b31117fdff3c391f5ff019e4bf470d9c58ca98eeb1ae3ad1dec3d5e350f2dc9d18e0b6b5b38846caace16ec6c
|
7
|
+
data.tar.gz: 0e9a871c24819b6df8b9d3911161c675ab3b167fa83ba74f95774ca1a3076136ae1114b2b8973233a1fdc44e716fd6d4c3f98ab52defc68518de04bfa831f2a7
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data.tar.gz.sig
CHANGED
Binary file
|
data/lib/feeds.xsl
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
2
|
+
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
|
3
|
+
<xsl:output method="html" indent="yes" />
|
4
|
+
|
5
|
+
<xsl:template match='feeds'>
|
6
|
+
<xsl:element name="div">
|
7
|
+
<xsl:apply-templates select='summary'/>
|
8
|
+
<xsl:apply-templates select='records'/>
|
9
|
+
</xsl:element>
|
10
|
+
</xsl:template>
|
11
|
+
|
12
|
+
<xsl:template match='feeds/summary'>
|
13
|
+
<h1><xsl:value-of select='title'/></h1>
|
14
|
+
</xsl:template>
|
15
|
+
|
16
|
+
<xsl:template match='records/column'>
|
17
|
+
|
18
|
+
<xsl:apply-templates select='summary'/>
|
19
|
+
<xsl:apply-templates select='records'/>
|
20
|
+
|
21
|
+
</xsl:template>
|
22
|
+
|
23
|
+
<xsl:template match='column/summary'>
|
24
|
+
<h2><xsl:value-of select='id'/></h2>
|
25
|
+
</xsl:template>
|
26
|
+
|
27
|
+
<xsl:template match='records/feed'>
|
28
|
+
<xsl:apply-templates select='summary'/>
|
29
|
+
<ul><xsl:apply-templates select='records'/></ul>
|
30
|
+
</xsl:template>
|
31
|
+
|
32
|
+
<xsl:template match='feed/summary'>
|
33
|
+
<h3><xsl:value-of select='title'/></h3>
|
34
|
+
</xsl:template>
|
35
|
+
|
36
|
+
<xsl:template match='records/item'>
|
37
|
+
<li>
|
38
|
+
<xsl:apply-templates select='summary'/>
|
39
|
+
<xsl:apply-templates select='records'/>
|
40
|
+
</li>
|
41
|
+
</xsl:template>
|
42
|
+
|
43
|
+
<xsl:template match='item/summary'>
|
44
|
+
<h4><xsl:value-of disable-output-escaping="yes" select='title'/></h4>
|
45
|
+
<p><xsl:value-of disable-output-escaping="yes" select='description'/></p>
|
46
|
+
</xsl:template>
|
47
|
+
|
48
|
+
</xsl:stylesheet>
|
data/lib/polyrex-feed-reader.rb
CHANGED
@@ -1,196 +1,88 @@
|
|
1
|
-
#!/usr/bin/ruby
|
1
|
+
#!/usr/bin/env ruby
|
2
2
|
|
3
3
|
# file: polyrex-feed-reader.rb
|
4
4
|
|
5
|
+
require 'nokogiri'
|
6
|
+
require 'rss_to_dynarex'
|
5
7
|
require 'polyrex'
|
6
|
-
require 'open-uri'
|
7
|
-
require 'rexml/document'
|
8
|
-
require 'builder'
|
9
|
-
require 'time'
|
10
|
-
require 'date'
|
11
|
-
require 'chronic'
|
12
|
-
require 'pathname'
|
13
|
-
|
14
|
-
class Fixnum
|
15
|
-
def seconds() self end
|
16
|
-
def minutes() self * 60 end
|
17
|
-
def hours() self * 60 * 60 end
|
18
|
-
def days() self * 3600 * 24 end
|
19
|
-
def weeks() self * 3600 * 24 * 7 end
|
20
|
-
def months() self * 86400 * 30 end
|
21
|
-
alias second seconds; alias hour hours; alias minute minutes
|
22
|
-
alias day days; alias week weeks; alias month months
|
23
|
-
end
|
24
|
-
|
25
8
|
|
26
9
|
class PolyrexFeedReader
|
27
|
-
include REXML
|
28
|
-
|
29
|
-
# read the Polyrex file otherwise create the file_path
|
30
|
-
#
|
31
|
-
def initialize(file_path)
|
32
|
-
@file_path = file_path
|
33
|
-
if File.exists? @file_path then
|
34
|
-
@feeds = Polyrex.new @file_path
|
35
|
-
else
|
36
|
-
schema = 'feeds/column[id]/feed[rss_url,title,important,occurrence,recent,url, xhtml, xhtml_mobile, last_modified, last_accessed]/item[title,link,description]'
|
37
|
-
@feeds = Polyrex.new schema
|
38
|
-
@feeds.save @file_path
|
39
|
-
end
|
40
|
-
end
|
41
10
|
|
42
|
-
|
43
|
-
#
|
44
|
-
def parse(lines)
|
45
|
-
@feeds.parse(lines)
|
46
|
-
end
|
11
|
+
def initialize(polyrex)
|
47
12
|
|
48
|
-
|
49
|
-
#
|
50
|
-
def read()
|
51
|
-
@feeds.records.each do |col|
|
52
|
-
col.records.each do |feed|
|
53
|
-
fetch_feed(feed)
|
54
|
-
end
|
55
|
-
col.records.sort_by!{|x| -Time.parse(x.text('summary/last_modified').to_s).to_i}
|
13
|
+
@polyrex = polyrex
|
56
14
|
|
57
|
-
|
15
|
+
end
|
58
16
|
|
59
|
-
|
17
|
+
def fetch_feeds()
|
60
18
|
|
61
|
-
|
62
|
-
interleaved = a[0].zip(*a[1..-1]).flatten
|
19
|
+
feeds = @polyrex.xpath('//column/records/feed/summary')
|
63
20
|
|
64
|
-
|
65
|
-
xml.instruct! :xml, :version => "1.0", :encoding => "UTF-8"
|
21
|
+
feeds.each do |feed|
|
66
22
|
|
67
|
-
|
68
|
-
xml.summary do
|
69
|
-
xml.recordx_type 'polyrex'
|
70
|
-
end
|
71
|
-
xml.records do
|
72
|
-
interleaved.each do |feed|
|
73
|
-
xml.feed do
|
74
|
-
xml.summary do
|
75
|
-
xml.title feed.title
|
76
|
-
xml.last_modified feed.last_modified
|
77
|
-
xml.recent feed.recent
|
78
|
-
xml.important feed.important
|
79
|
-
end
|
80
|
-
xml.records do
|
81
|
-
feed.records.each do |item|
|
82
|
-
xml.item do
|
83
|
-
xml.summary do
|
84
|
-
xml.title item.title
|
85
|
-
xml.description item.description
|
86
|
-
end
|
87
|
-
xml.records
|
88
|
-
end
|
89
|
-
end #
|
90
|
-
end # / records
|
91
|
-
end # /feed
|
92
|
-
end
|
93
|
-
end # /records
|
94
|
-
end
|
23
|
+
puts "fetching %s ..." % feed.text('rss_url').inspect
|
95
24
|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
def read_feed(feed_name)
|
102
|
-
feed = @feeds.find_by_feed_title(feed_name)
|
103
|
-
fetch_feed(feed, save_rss: true)
|
104
|
-
@feeds.save @file_path
|
105
|
-
|
106
|
-
File.open(rss_file(feed),'r').read
|
25
|
+
rtd = RSStoDynarex.new feed.text('rss_url')
|
26
|
+
dynarex = rtd.to_dynarex
|
27
|
+
dynarex.save "%s.xml" % feed.text('title')\
|
28
|
+
.downcase.gsub(/\s/,'').gsub('-','_')
|
29
|
+
end
|
107
30
|
end
|
108
31
|
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
o = {save_rss: false}.merge opts
|
113
|
-
|
114
|
-
if scheduled? feed.occurrence and recency(feed.last_accessed) != 'hot' then
|
115
|
-
|
116
|
-
buffer = open(feed.rss_url, 'UserAgent' => 'PolyrexFeedReader').read
|
117
|
-
if o[:save_rss] == true then
|
118
|
-
|
119
|
-
pathname = Pathname.new(rss_file(feed))
|
120
|
-
pathname.dirname.mkdir unless pathname.dirname.exist?
|
121
|
-
File.open(rss_file(feed),'w'){|f| f.write buffer}
|
122
|
-
|
123
|
-
end
|
124
|
-
rss_doc = Document.new(buffer)
|
125
|
-
rss_items = XPath.match(rss_doc.root, '//item')[0..2]
|
32
|
+
def refresh
|
33
|
+
|
34
|
+
@polyrex.records.each do |column|
|
126
35
|
|
127
|
-
|
128
|
-
# create the items
|
129
|
-
k = 3 - feed.records.length
|
130
|
-
k.times {|x| feed.create.item}
|
36
|
+
column.records.each do |feed|
|
131
37
|
|
132
|
-
|
38
|
+
filename = "%s.xml" % feed.title\
|
39
|
+
.downcase.gsub(/\s/,'').gsub('-','_')
|
133
40
|
|
134
|
-
|
41
|
+
d = Dynarex.new filename
|
135
42
|
|
136
|
-
|
43
|
+
d.to_h[0..2].each.with_index do |x, i|
|
137
44
|
|
138
|
-
|
139
|
-
|
140
|
-
|
45
|
+
h = {title: x[:title]}
|
46
|
+
|
47
|
+
if i == 0 then
|
48
|
+
|
49
|
+
raw_desc = CGI.unescapeHTML(x[:description]).gsub(/<\/?[^>]*>/, "")
|
50
|
+
desc = raw_desc.length > 300 ? raw_desc[0..296] + ' ...' : raw_desc
|
51
|
+
h[:description] = desc
|
52
|
+
end
|
53
|
+
|
54
|
+
feed.create.item h
|
141
55
|
end
|
142
|
-
end
|
143
|
-
feed.last_accessed = Time.now
|
144
|
-
end
|
145
|
-
end
|
146
56
|
|
147
|
-
|
148
|
-
case (Time.now - Time.parse(time))
|
149
|
-
when 1.second..5.minutes then 'hot'
|
150
|
-
when 5.minutes..4.hours then 'warm'
|
151
|
-
when 4.hours..1.week then 'cold'
|
152
|
-
when 1.week..1.month then 'coldx1week'
|
153
|
-
when 1.month..6.months then 'coldx1month'
|
154
|
-
else 'coldx6months'
|
57
|
+
end
|
155
58
|
end
|
156
59
|
end
|
157
60
|
|
158
|
-
|
159
|
-
important = feed.important.downcase == 'important'
|
160
|
-
feed.last_modified = Time.now
|
161
|
-
feed.recent = 'hot'
|
61
|
+
alias update_doc refresh
|
162
62
|
|
163
|
-
|
164
|
-
feed.item[i].title = rss_item.text('title')
|
165
|
-
end
|
166
|
-
feed.item[0].description = rss_items[0].text('description') if important
|
167
|
-
end
|
63
|
+
def save_html(filepath='feeds.html')
|
168
64
|
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
d = Time.now.wday
|
174
|
-
pattern = "%s|%s" % [Date::DAYNAMES[d],Date::ABBR_DAYNAMES[d]]
|
175
|
-
a.map!{|x| x.sub(/#{pattern}/i,'today')}
|
176
|
-
|
177
|
-
dates = a.map do |s|
|
178
|
-
if s.split(/\s/).length > 1 then
|
179
|
-
d = Chronic.parse(s, guess: false)
|
180
|
-
[d.first, d.last]
|
181
|
-
else
|
182
|
-
d = Chronic.parse(s)
|
183
|
-
d1 = d2 = Time.parse("%s-%s-%s" % [d.year, d.month, d.day])
|
184
|
-
[d1, d2 + 24.hours]
|
185
|
-
end
|
186
|
-
end
|
65
|
+
lib = File.dirname(__FILE__)
|
66
|
+
xsl_buffer = File.read(lib + '/feeds.xsl')
|
67
|
+
#xsl_buffer = File.read('feeds.xsl')
|
187
68
|
|
188
|
-
|
69
|
+
xslt = Nokogiri::XSLT(xsl_buffer)
|
70
|
+
html = xslt.transform(Nokogiri::XML(@polyrex.to_xml)).to_s
|
71
|
+
File.write filepath, html
|
189
72
|
end
|
190
|
-
|
191
|
-
def
|
192
|
-
|
73
|
+
|
74
|
+
def save_xml(filepath='feeds.xml')
|
75
|
+
@polyrex.save filepath, pretty: true
|
193
76
|
end
|
194
77
|
|
195
78
|
end
|
196
79
|
|
80
|
+
if __FILE__ == $0 then
|
81
|
+
|
82
|
+
pfr = PolyrexFeedReader.new(px)
|
83
|
+
pfr.fetch_feeds
|
84
|
+
pfr.update_doc
|
85
|
+
pfr.save_xml 'feeds.xml'
|
86
|
+
pfr.save_html 'feeds.html'
|
87
|
+
|
88
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: polyrex-feed-reader
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- James Robertson
|
@@ -29,48 +29,48 @@ cert_chain:
|
|
29
29
|
LxhpHfLSRfOWDFimU0YGVs8dISSHOneULhblAYTTb7tvv2Zr8wG/bdlLhWuwoz23
|
30
30
|
L5EDjzznPMmVNG60mdktue6PKdOEFp7N
|
31
31
|
-----END CERTIFICATE-----
|
32
|
-
date: 2013-07
|
32
|
+
date: 2013-12-07 00:00:00.000000000 Z
|
33
33
|
dependencies:
|
34
34
|
- !ruby/object:Gem::Dependency
|
35
35
|
name: polyrex
|
36
36
|
requirement: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
type: :runtime
|
42
42
|
prerelease: false
|
43
43
|
version_requirements: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- -
|
45
|
+
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: '0'
|
48
48
|
- !ruby/object:Gem::Dependency
|
49
|
-
name:
|
49
|
+
name: rss_to_dynarex
|
50
50
|
requirement: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- -
|
52
|
+
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
55
|
type: :runtime
|
56
56
|
prerelease: false
|
57
57
|
version_requirements: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- -
|
59
|
+
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: '0'
|
62
62
|
- !ruby/object:Gem::Dependency
|
63
|
-
name:
|
63
|
+
name: nokogiri
|
64
64
|
requirement: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- -
|
66
|
+
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
69
|
type: :runtime
|
70
70
|
prerelease: false
|
71
71
|
version_requirements: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- -
|
73
|
+
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
75
|
version: '0'
|
76
76
|
description:
|
@@ -80,6 +80,7 @@ extensions: []
|
|
80
80
|
extra_rdoc_files: []
|
81
81
|
files:
|
82
82
|
- lib/polyrex-feed-reader.rb
|
83
|
+
- lib/feeds.xsl
|
83
84
|
homepage: https://github.com/jrobertson/polyrex-feed-reader
|
84
85
|
licenses:
|
85
86
|
- MIT
|
@@ -90,12 +91,12 @@ require_paths:
|
|
90
91
|
- lib
|
91
92
|
required_ruby_version: !ruby/object:Gem::Requirement
|
92
93
|
requirements:
|
93
|
-
- -
|
94
|
+
- - ">="
|
94
95
|
- !ruby/object:Gem::Version
|
95
96
|
version: '0'
|
96
97
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
97
98
|
requirements:
|
98
|
-
- -
|
99
|
+
- - ">="
|
99
100
|
- !ruby/object:Gem::Version
|
100
101
|
version: '0'
|
101
102
|
requirements: []
|
@@ -103,5 +104,6 @@ rubyforge_project:
|
|
103
104
|
rubygems_version: 2.0.0.rc.2
|
104
105
|
signing_key:
|
105
106
|
specification_version: 4
|
106
|
-
summary:
|
107
|
+
summary: Fetches RSS feeds from a Polyrex file and displays each feed summary on an
|
108
|
+
HTML page
|
107
109
|
test_files: []
|
metadata.gz.sig
CHANGED
Binary file
|