audrey2 0.1.1 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/audrey2.conf.sample +23 -2
- data/audrey2.gemspec +3 -3
- data/lib/audrey2.rb +136 -83
- metadata +5 -5
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.2.0
|
data/audrey2.conf.sample
CHANGED
@@ -12,13 +12,34 @@
|
|
12
12
|
recipes_folder: ./recipes
|
13
13
|
themes_folder: ./themes
|
14
14
|
|
15
|
+
#
|
15
16
|
# User agent to impersonate when retrieving themes. Please use with discretion.
|
16
|
-
#
|
17
|
+
#
|
18
|
+
# user_agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3 (.NET CLR 1.1.4322)
|
19
|
+
|
20
|
+
#
|
21
|
+
# Email notification of exceptions
|
22
|
+
#
|
23
|
+
# This feature requires Mikel Lindsaar's 'mail' gem (http://github.com/mikel/mail)
|
24
|
+
#
|
25
|
+
# If you want to use sendmail instead of smtp then set via this way:
|
26
|
+
# via: sendmail
|
27
|
+
# and omit the smtp hash.
|
28
|
+
#
|
29
|
+
# email:
|
30
|
+
# to: test@test.com
|
31
|
+
# from: Audrey 2.0 <noreply@test.com>
|
32
|
+
# via: smtp
|
33
|
+
# smtp:
|
34
|
+
# server: smtp.test.com
|
35
|
+
# port: 25
|
36
|
+
# user_name: test
|
37
|
+
# password: password
|
38
|
+
# domain: test.com
|
17
39
|
|
18
40
|
#
|
19
41
|
# Not yet implemented:
|
20
42
|
#
|
21
|
-
# email_errors_to: test@test.com
|
22
43
|
# log_file: /var/log/audrey2.log
|
23
44
|
# log_level: debug
|
24
45
|
|
data/audrey2.gemspec
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
# Generated by jeweler
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
-
# Instead, edit Jeweler::Tasks in
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{audrey2}
|
8
|
-
s.version = "0.
|
8
|
+
s.version = "0.2.0"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Sven Aas"]
|
12
|
-
s.date = %q{2010-07-
|
12
|
+
s.date = %q{2010-07-30}
|
13
13
|
s.default_executable = %q{feedme}
|
14
14
|
s.description = %q{Gem for feed processing and aggregation}
|
15
15
|
s.email = %q{sven.aas@gmail.com}
|
data/lib/audrey2.rb
CHANGED
@@ -4,166 +4,219 @@ require 'feed-normalizer'
|
|
4
4
|
require 'open-uri'
|
5
5
|
require 'haml'
|
6
6
|
|
7
|
-
module Audrey2
|
7
|
+
module Audrey2
|
8
8
|
|
9
9
|
class Aggregator
|
10
10
|
def initialize(configfile)
|
11
11
|
init_config(configfile)
|
12
12
|
end
|
13
|
-
|
13
|
+
|
14
14
|
def feed_me(recipe_name)
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
15
|
+
begin
|
16
|
+
# Load recipe and theme and make sure everything is in order
|
17
|
+
recipe = load_recipe(recipe_name)
|
18
|
+
init_theme(recipe['theme'])
|
19
|
+
output_file = recipe['output_file']
|
20
|
+
verify_output_file(output_file)
|
21
|
+
max_entries = recipe['max_entries'] || 1000
|
22
|
+
|
23
|
+
# Download and parse the feeds
|
24
|
+
entry_sources = {}
|
25
|
+
feeds = recipe['feeds'].collect { |feed| parse_feed(feed, entry_sources) }
|
26
|
+
|
27
|
+
# Aggregate and sort the entries
|
28
|
+
entries = []
|
29
|
+
feeds.each { |feed| entries += feed.entries }
|
30
|
+
sort!(entries)
|
31
|
+
|
32
|
+
# Prepare template evaluation scope including any helper code defined in the theme
|
33
|
+
scope = Object.new
|
34
|
+
scope.instance_eval(@helper_code) if @helper_code
|
35
|
+
|
36
|
+
# Output the aggregated entries
|
37
|
+
output = ''
|
38
|
+
engine ||= Haml::Engine.new(@entry_template)
|
39
|
+
|
40
|
+
entries[0..max_entries - 1].each do |entry|
|
41
|
+
output << engine.render(scope, :entry => entry, :source => entry_sources[entry])
|
42
|
+
end
|
43
|
+
|
44
|
+
File.open(output_file, 'w') { |f| f << output }
|
45
|
+
|
46
|
+
rescue Exception => e
|
47
|
+
$stderr.puts "An exception occurred while running recipe #{recipe_name}:\n\n#{e}\n#{e.backtrace}"
|
48
|
+
if @email
|
49
|
+
email(<<-EOF
|
50
|
+
An exception occurred while running recipe #{recipe_name}
|
51
|
+
|
52
|
+
Exception: #{e}
|
53
|
+
|
54
|
+
Backtrace:
|
55
|
+
|
56
|
+
#{e.backtrace}
|
57
|
+
EOF
|
58
|
+
)
|
59
|
+
else
|
60
|
+
exit(1)
|
61
|
+
end
|
41
62
|
end
|
42
|
-
|
43
|
-
File.open(output_file, 'w') { |f| f << output }
|
44
63
|
end
|
45
|
-
|
64
|
+
|
46
65
|
protected
|
66
|
+
def email(e)
|
67
|
+
return unless @email
|
68
|
+
|
69
|
+
mail = Mail.new
|
70
|
+
mail[:from] = @email['from']
|
71
|
+
mail[:to] = @email['to']
|
72
|
+
mail[:subject] = "[AUDREY 2.0] Exception Notification"
|
73
|
+
mail[:body] = e
|
74
|
+
|
75
|
+
case @email['via']
|
76
|
+
when 'sendmail'
|
77
|
+
mail.delivery_method :sendmail
|
78
|
+
when 'smtp'
|
79
|
+
raise "Missing SMTP configuration" unless @email['smtp']
|
80
|
+
smtp = {
|
81
|
+
:address => @email['smtp']['server'] || 'localhost',
|
82
|
+
:port => @email['smtp']['port'] || 25
|
83
|
+
}
|
84
|
+
smtp[:domain] = @email['smtp']['domain'] if @email['smtp']['domain']
|
85
|
+
smtp[:user_name] = @email['smtp']['user_name'] if @email['smtp']['user_name']
|
86
|
+
smtp[:password] = @email['smtp']['password'] if @email['smtp']['password']
|
87
|
+
smtp[:authentication] = @email['smtp']['authentication'] if @email['smtp']['authentication']
|
88
|
+
mail.delivery_method :smtp, smtp
|
89
|
+
end
|
90
|
+
|
91
|
+
mail.deliver
|
92
|
+
end
|
93
|
+
|
47
94
|
def verify_output_file(output_file)
|
48
95
|
output_folder = File.dirname(output_file)
|
49
96
|
if ! File.exist? output_folder
|
50
|
-
|
51
|
-
exit(1)
|
97
|
+
raise "ERROR: Output folder #{output_folder} does not exist."
|
52
98
|
elsif ! File.writable? output_folder
|
53
|
-
|
54
|
-
|
55
|
-
end
|
99
|
+
raise "ERROR: Output folder #{output_folder} is not writable"
|
100
|
+
end
|
56
101
|
if File.exist?(output_file)
|
57
102
|
if ! File.writable? output_file
|
58
|
-
|
59
|
-
exit(1)
|
103
|
+
raise "ERROR: Output file #{output_file} is not writable"
|
60
104
|
end
|
61
105
|
end
|
62
106
|
end
|
63
|
-
|
107
|
+
|
64
108
|
def init_theme(theme)
|
65
109
|
theme_path = File.join(@themes_folder, theme)
|
66
110
|
if ! File.exist? theme_path
|
67
|
-
$stderr.puts "ERROR: Theme #{theme_path} does not exist."
|
68
|
-
exit(1)
|
111
|
+
$stderr.puts "ERROR: Theme #{theme_path} does not exist."
|
112
|
+
exit(1)
|
69
113
|
elsif ! File.readable? theme_path
|
70
114
|
$stderr.puts "ERROR: Theme #{theme_path} is not readable"
|
71
|
-
exit(1)
|
72
|
-
end
|
73
|
-
|
115
|
+
exit(1)
|
116
|
+
end
|
117
|
+
|
74
118
|
entry_template_file = File.join(@themes_folder, theme, 'entry.haml')
|
75
119
|
if ! File.exist? entry_template_file
|
76
|
-
$stderr.puts "ERROR: Theme #{theme} does not include an entry template (entry.haml)"
|
77
|
-
exit(1)
|
120
|
+
$stderr.puts "ERROR: Theme #{theme} does not include an entry template (entry.haml)"
|
121
|
+
exit(1)
|
78
122
|
elsif ! File.readable? entry_template_file
|
79
123
|
$stderr.puts "ERROR: Entry template #{entry_template_file} is not readable"
|
80
|
-
exit(1)
|
81
|
-
end
|
124
|
+
exit(1)
|
125
|
+
end
|
82
126
|
@entry_template = File.read(entry_template_file)
|
83
|
-
|
127
|
+
|
84
128
|
helper_file = File.join(@themes_folder, theme, 'helpers.rb')
|
85
129
|
@helper_code = nil
|
86
130
|
if File.exist? helper_file
|
87
131
|
if ! File.readable? helper_file
|
88
132
|
$stderr.puts "ERROR: Helper file #{helper_file} is not readable"
|
89
|
-
exit(1)
|
133
|
+
exit(1)
|
90
134
|
end
|
91
|
-
@helper_code = File.open(helper_file) { |f| f.read }
|
92
|
-
end
|
93
|
-
end
|
94
|
-
|
95
|
-
# Uses the sort order specified in configuration
|
135
|
+
@helper_code = File.open(helper_file) { |f| f.read }
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
# Uses the sort order specified in configuration
|
96
140
|
def sort!(entries)
|
97
141
|
case @sort
|
98
142
|
when 'reverse-chronological'
|
99
|
-
entries.sort! {|a, b| b.date_published <=> a.date_published } # Reverse chronological
|
143
|
+
entries.sort! {|a, b| b.date_published <=> a.date_published } # Reverse chronological
|
100
144
|
end
|
101
145
|
end
|
102
|
-
|
146
|
+
|
103
147
|
def parse_feed(feed, entry_sources)
|
104
148
|
remote_feed = nil
|
105
149
|
begin
|
106
|
-
remote_feed = open(feed['url'], "User-Agent" => @user_agent)
|
107
|
-
rescue Exception => e
|
150
|
+
remote_feed = open(feed['url'], "User-Agent" => @user_agent)
|
151
|
+
rescue Exception => e
|
108
152
|
raise "Exception occurred when opening feed #{feed['name']} at #{feed['url']}:\n\n" + e.to_s
|
109
153
|
end
|
110
154
|
|
111
155
|
parsed_feed = nil
|
112
156
|
begin
|
113
157
|
parsed_feed = FeedNormalizer::FeedNormalizer.parse remote_feed
|
114
|
-
rescue Exception => e
|
115
|
-
raise "Exception occurred when parsing feed #{feed['name']} which was downloaded from #{feed['url']}:\n\n" + e.to_s
|
158
|
+
rescue Exception => e
|
159
|
+
raise "Exception occurred when parsing feed #{feed['name']} which was downloaded from #{feed['url']}:\n\n" + e.to_s
|
116
160
|
end
|
117
161
|
|
162
|
+
raise "Feed #{feed['name']} at #{feed['url']} does not appear to be a parsable feed" unless parsed_feed
|
163
|
+
|
118
164
|
parsed_feed.entries.each { |entry| entry_sources[entry] = feed }
|
119
|
-
|
120
|
-
parsed_feed
|
165
|
+
|
166
|
+
parsed_feed
|
121
167
|
end
|
122
|
-
|
168
|
+
|
123
169
|
def load_recipe(recipe)
|
124
170
|
recipefile = File.join(@recipes_folder, recipe)
|
125
171
|
if ! File.exist? recipefile
|
126
|
-
$stderr.puts "ERROR: Recipe #{recipefile} does not exist"
|
127
|
-
exit(1)
|
172
|
+
$stderr.puts "ERROR: Recipe #{recipefile} does not exist"
|
173
|
+
exit(1)
|
128
174
|
elsif ! File.readable? recipefile
|
129
175
|
$stderr.puts "ERROR: Recipe file #{recipefile} is not readable"
|
130
|
-
exit(1)
|
176
|
+
exit(1)
|
131
177
|
end
|
132
178
|
YAML::load_file(recipefile)
|
133
179
|
end
|
134
|
-
|
180
|
+
|
135
181
|
def init_config(configfile)
|
136
182
|
if ! File.exist? configfile
|
137
|
-
$stderr.puts "ERROR: Configuration file #{configfile} does not exist"
|
138
|
-
exit(1)
|
183
|
+
$stderr.puts "ERROR: Configuration file #{configfile} does not exist"
|
184
|
+
exit(1)
|
139
185
|
elsif ! File.readable? configfile
|
140
186
|
$stderr.puts "ERROR: Configuration file #{configfile} is not readable"
|
141
|
-
exit(1)
|
187
|
+
exit(1)
|
142
188
|
end
|
143
|
-
|
189
|
+
|
144
190
|
config = YAML::load_file(configfile)
|
145
|
-
|
191
|
+
|
146
192
|
@recipes_folder = config['recipes_folder']
|
147
193
|
if ! File.exist? @recipes_folder
|
148
|
-
$stderr.puts "ERROR: Recipes folder #{@recipes_folder} does not exist"
|
149
|
-
exit(1)
|
194
|
+
$stderr.puts "ERROR: Recipes folder #{@recipes_folder} does not exist"
|
195
|
+
exit(1)
|
150
196
|
elsif ! File.readable? @recipes_folder
|
151
197
|
$stderr.puts "ERROR: Recipes folder #{@recipes_folder} is not readable"
|
152
|
-
exit(1)
|
198
|
+
exit(1)
|
153
199
|
end
|
154
|
-
|
200
|
+
|
155
201
|
@themes_folder = config['themes_folder']
|
156
202
|
if ! File.exist? @themes_folder
|
157
|
-
$stderr.puts "ERROR: Themes folder #{@themes_folder} does not exist"
|
158
|
-
exit(1)
|
203
|
+
$stderr.puts "ERROR: Themes folder #{@themes_folder} does not exist"
|
204
|
+
exit(1)
|
159
205
|
elsif ! File.readable? @themes_folder
|
160
206
|
$stderr.puts "ERROR: Themes folder #{@themes_folder} is not readable"
|
161
|
-
exit(1)
|
207
|
+
exit(1)
|
162
208
|
end
|
163
|
-
|
209
|
+
|
164
210
|
@user_agent = config['user_agent'] || 'Audrey 2.0 Feed Aggregator'
|
165
211
|
@sort = config['sort'] || 'reverse-chronological'
|
212
|
+
|
213
|
+
if @email = config['email']
|
214
|
+
gem 'mail', '~> 2.2.5'
|
215
|
+
require 'mail'
|
216
|
+
# TODO: Check for required/consistent email config
|
217
|
+
end
|
218
|
+
|
166
219
|
end
|
167
|
-
end
|
168
|
-
|
220
|
+
end
|
221
|
+
|
169
222
|
end
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: audrey2
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 23
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 0
|
8
|
-
-
|
9
|
-
-
|
10
|
-
version: 0.
|
8
|
+
- 2
|
9
|
+
- 0
|
10
|
+
version: 0.2.0
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Sven Aas
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2010-07-
|
18
|
+
date: 2010-07-30 00:00:00 -04:00
|
19
19
|
default_executable: feedme
|
20
20
|
dependencies:
|
21
21
|
- !ruby/object:Gem::Dependency
|