feedtools 0.2.8 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG +6 -0
- data/lib/database_feed_cache.rb +145 -0
- data/lib/feed_tools.rb +91 -191
- data/rakefile +2 -1
- metadata +13 -2
data/CHANGELOG
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
== FeedTools 0.2.9
|
|
2
|
+
* changed ordering of elements checked for the item content
|
|
3
|
+
* added dependancy on uuidtools, uuids used in feed generation
|
|
4
|
+
* database feed cache implementation factored out into its own file
|
|
5
|
+
* http_fetch changed to use lambda
|
|
6
|
+
* fixed invalid rss 1.0/2.0 generation (missing some namespaces)
|
|
1
7
|
== FeedTools 0.2.8
|
|
2
8
|
* fixed bug with http headers being loaded from invalid YAML
|
|
3
9
|
* fixed uninitialized constant bug (you shouldn't have RUBYOPT set)
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
#= database_feed_cache.rb
|
|
2
|
+
#
|
|
3
|
+
# The <tt>DatabaseFeedCache</tt> is the default caching mechanism for
|
|
4
|
+
# FeedTools. This mechanism can be replaced easily by creating another
|
|
5
|
+
# class with the required set of methods and setting
|
|
6
|
+
# <tt>FeedTools#feed_cache</tt> to the new class.
|
|
7
|
+
module FeedTools
|
|
8
|
+
# The default caching mechanism for the FeedTools module
|
|
9
|
+
class DatabaseFeedCache < ActiveRecord::Base
|
|
10
|
+
# Overrides the default table name to use the "feeds" table.
|
|
11
|
+
def self.table_name() "feeds" end
|
|
12
|
+
|
|
13
|
+
# If ActiveRecord is not already connected, attempts to find a configuration file and use
|
|
14
|
+
# it to open a connection for ActiveRecord.
|
|
15
|
+
# This method is probably unnecessary for anything but testing and debugging purposes.
|
|
16
|
+
# In a Rails environment, the connection will already have been established
|
|
17
|
+
# and this method will simply do nothing.
|
|
18
|
+
#
|
|
19
|
+
# This method should not raise any exceptions because it's designed to be run only when
|
|
20
|
+
# the module is first loaded. If it fails, the user should get an exception when they
|
|
21
|
+
# try to perform some action that makes use of the caching functionality, and not until.
|
|
22
|
+
def DatabaseFeedCache.initialize_cache
|
|
23
|
+
# Establish a connection if we don't already have one
|
|
24
|
+
begin
|
|
25
|
+
ActiveRecord::Base.connection
|
|
26
|
+
rescue
|
|
27
|
+
begin
|
|
28
|
+
possible_config_files = [
|
|
29
|
+
"./config/database.yml",
|
|
30
|
+
"../config/database.yml",
|
|
31
|
+
"./database.yml",
|
|
32
|
+
"../database.yml"
|
|
33
|
+
]
|
|
34
|
+
database_config_file = nil
|
|
35
|
+
for file in possible_config_files
|
|
36
|
+
if File.exists? file
|
|
37
|
+
database_config_file = file
|
|
38
|
+
break
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
database_config_hash = File.open(database_config_file) do |file|
|
|
42
|
+
config_hash = YAML::load(file)
|
|
43
|
+
unless config_hash[FEED_TOOLS_ENV].nil?
|
|
44
|
+
config_hash = config_hash[FEED_TOOLS_ENV]
|
|
45
|
+
end
|
|
46
|
+
config_hash
|
|
47
|
+
end
|
|
48
|
+
ActiveRecord::Base.configurations = database_config_hash
|
|
49
|
+
ActiveRecord::Base.establish_connection(database_config_hash)
|
|
50
|
+
ActiveRecord::Base.connection
|
|
51
|
+
rescue
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
# Verify that the necessary database tables are in place
|
|
55
|
+
# and if they're missing, create them
|
|
56
|
+
unless DatabaseFeedCache.table_exists?
|
|
57
|
+
DatabaseFeedCache.create_table
|
|
58
|
+
end
|
|
59
|
+
return nil
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Returns true if a connection to the database has been established and the
|
|
63
|
+
# required table structure is in place.
|
|
64
|
+
def DatabaseFeedCache.connected?
|
|
65
|
+
begin
|
|
66
|
+
ActiveRecord::Base.connection
|
|
67
|
+
return false if ActiveRecord::Base.configurations.nil?
|
|
68
|
+
return false unless DatabaseFeedCache.table_exists?
|
|
69
|
+
rescue => error
|
|
70
|
+
return false
|
|
71
|
+
end
|
|
72
|
+
return true
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# True if the appropriate database table already exists
|
|
76
|
+
def DatabaseFeedCache.table_exists?
|
|
77
|
+
begin
|
|
78
|
+
ActiveRecord::Base.connection.execute "select id, url, title, " +
|
|
79
|
+
"link, xml_data, http_headers, last_retrieved " +
|
|
80
|
+
"from #{self.table_name()} limit 1"
|
|
81
|
+
rescue ActiveRecord::StatementInvalid
|
|
82
|
+
return false
|
|
83
|
+
rescue
|
|
84
|
+
return false
|
|
85
|
+
end
|
|
86
|
+
return true
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Creates the appropriate database table
|
|
90
|
+
# Deprecated.
|
|
91
|
+
def DatabaseFeedCache.create_table
|
|
92
|
+
warn("The table creation methods will be removed in a future version " +
|
|
93
|
+
"and any reliance on this method should be removed.")
|
|
94
|
+
unless DatabaseFeedCache.table_exists?
|
|
95
|
+
feeds_mysql = <<-SQL_END
|
|
96
|
+
CREATE TABLE `#{self.table_name()}` (
|
|
97
|
+
`id` int(10) unsigned NOT NULL auto_increment,
|
|
98
|
+
`url` varchar(255) default NULL,
|
|
99
|
+
`title` varchar(255) default NULL,
|
|
100
|
+
`link` varchar(255) default NULL,
|
|
101
|
+
`xml_data` longtext default NULL,
|
|
102
|
+
`http_headers` text default NULL,
|
|
103
|
+
`last_retrieved` datetime default NULL,
|
|
104
|
+
PRIMARY KEY (`id`)
|
|
105
|
+
)
|
|
106
|
+
SQL_END
|
|
107
|
+
feeds_sqlite = <<-SQL_END
|
|
108
|
+
CREATE TABLE '#{self.table_name()}' (
|
|
109
|
+
'id' INTEGER PRIMARY KEY NOT NULL,
|
|
110
|
+
'url' VARCHAR(255) DEFAULT NULL,
|
|
111
|
+
'title' VARCHAR(255) DEFAULT NULL,
|
|
112
|
+
'link' VARCHAR(255) DEFAULT NULL,
|
|
113
|
+
'xml_data' TEXT DEFAULT NULL,
|
|
114
|
+
'http_headers' TEXT DEFAULT NULL,
|
|
115
|
+
'last_retrieved' DATETIME DEFAULT NULL
|
|
116
|
+
);
|
|
117
|
+
SQL_END
|
|
118
|
+
feeds_psql = <<-SQL_END
|
|
119
|
+
CREATE TABLE #{self.table_name()} (
|
|
120
|
+
id SERIAL PRIMARY KEY NOT NULL,
|
|
121
|
+
url varchar(255) default NULL,
|
|
122
|
+
title varchar(255) default NULL,
|
|
123
|
+
link varchar(255) default NULL,
|
|
124
|
+
xml_data text default NULL,
|
|
125
|
+
http_headers text default NULL,
|
|
126
|
+
last_retrieved timestamp default NULL
|
|
127
|
+
);
|
|
128
|
+
SQL_END
|
|
129
|
+
table_creation_sql = nil
|
|
130
|
+
if configurations["adapter"] == "mysql"
|
|
131
|
+
table_creation_sql = feeds_mysql
|
|
132
|
+
elsif configurations["adapter"] == "sqlite"
|
|
133
|
+
table_creation_sql = feeds_sqlite
|
|
134
|
+
elsif configurations["adapter"] == "postgresql"
|
|
135
|
+
table_creation_sql = feeds_psql
|
|
136
|
+
end
|
|
137
|
+
if table_creation_sql.nil?
|
|
138
|
+
raise "Could not build #{self.table_name()} table."
|
|
139
|
+
else
|
|
140
|
+
connection.execute table_creation_sql
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
end
|
data/lib/feed_tools.rb
CHANGED
|
@@ -22,19 +22,30 @@
|
|
|
22
22
|
#++
|
|
23
23
|
|
|
24
24
|
if Object.const_defined?(:FEED_TOOLS_ENV)
|
|
25
|
-
|
|
26
|
-
"by the presence of the RUBYOPT environment variable
|
|
25
|
+
warn("FeedTools may have been loaded improperly. This may be caused " +
|
|
26
|
+
"by the presence of the RUBYOPT environment variable or by using " +
|
|
27
|
+
"load instead of require. This can also be caused by missing " +
|
|
28
|
+
"the Iconv library, which is common on Windows.")
|
|
27
29
|
end
|
|
28
30
|
|
|
29
31
|
FEED_TOOLS_ENV = ENV['FEED_TOOLS_ENV'] ||
|
|
30
32
|
ENV['RAILS_ENV'] ||
|
|
31
33
|
'production' # :nodoc:
|
|
32
34
|
|
|
33
|
-
FEED_TOOLS_VERSION = "0.2.
|
|
35
|
+
FEED_TOOLS_VERSION = "0.2.9"
|
|
34
36
|
|
|
35
37
|
$:.unshift(File.dirname(__FILE__))
|
|
36
38
|
$:.unshift(File.dirname(__FILE__) + "/feed_tools/vendor")
|
|
37
|
-
|
|
39
|
+
|
|
40
|
+
begin
|
|
41
|
+
require 'iconv'
|
|
42
|
+
rescue LoadError
|
|
43
|
+
warn("The Iconv library does not appear to be installed properly. " +
|
|
44
|
+
"FeedTools cannot function properly without it.")
|
|
45
|
+
raise
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
require 'rubygems'
|
|
38
49
|
|
|
39
50
|
begin
|
|
40
51
|
require 'builder'
|
|
@@ -57,15 +68,16 @@ require 'net/ftp'
|
|
|
57
68
|
|
|
58
69
|
require 'rexml/document'
|
|
59
70
|
|
|
60
|
-
require 'iconv'
|
|
61
71
|
require 'uri'
|
|
62
72
|
require 'time'
|
|
63
73
|
require 'cgi'
|
|
64
74
|
require 'pp'
|
|
65
75
|
require 'yaml'
|
|
66
76
|
|
|
67
|
-
require 'rubygems'
|
|
68
77
|
require_gem('activerecord', '>= 1.10.1')
|
|
78
|
+
require_gem('uuidtools', '>= 0.1.1')
|
|
79
|
+
|
|
80
|
+
require 'database_feed_cache'
|
|
69
81
|
|
|
70
82
|
#= feed_tools.rb
|
|
71
83
|
#
|
|
@@ -84,141 +96,6 @@ require_gem('activerecord', '>= 1.10.1')
|
|
|
84
96
|
# => "43,37,28,23,11,3,1"
|
|
85
97
|
module FeedTools
|
|
86
98
|
|
|
87
|
-
# The default caching mechanism for the FeedTools module
|
|
88
|
-
class DatabaseFeedCache < ActiveRecord::Base
|
|
89
|
-
# Overrides the default table name to use the "feeds" table.
|
|
90
|
-
def self.table_name() "feeds" end
|
|
91
|
-
|
|
92
|
-
# If ActiveRecord is not already connected, attempts to find a configuration file and use
|
|
93
|
-
# it to open a connection for ActiveRecord.
|
|
94
|
-
# This method is probably unnecessary for anything but testing and debugging purposes.
|
|
95
|
-
# In a Rails environment, the connection will already have been established
|
|
96
|
-
# and this method will simply do nothing.
|
|
97
|
-
#
|
|
98
|
-
# This method should not raise any exceptions because it's designed to be run only when
|
|
99
|
-
# the module is first loaded. If it fails, the user should get an exception when they
|
|
100
|
-
# try to perform some action that makes use of the caching functionality, and not until.
|
|
101
|
-
def DatabaseFeedCache.initialize_cache
|
|
102
|
-
# Establish a connection if we don't already have one
|
|
103
|
-
begin
|
|
104
|
-
ActiveRecord::Base.connection
|
|
105
|
-
rescue
|
|
106
|
-
begin
|
|
107
|
-
possible_config_files = [
|
|
108
|
-
"./config/database.yml",
|
|
109
|
-
"../config/database.yml",
|
|
110
|
-
"./database.yml",
|
|
111
|
-
"../database.yml"
|
|
112
|
-
]
|
|
113
|
-
database_config_file = nil
|
|
114
|
-
for file in possible_config_files
|
|
115
|
-
if File.exists? file
|
|
116
|
-
database_config_file = file
|
|
117
|
-
break
|
|
118
|
-
end
|
|
119
|
-
end
|
|
120
|
-
database_config_hash = File.open(database_config_file) do |file|
|
|
121
|
-
config_hash = YAML::load(file)
|
|
122
|
-
unless config_hash[FEED_TOOLS_ENV].nil?
|
|
123
|
-
config_hash = config_hash[FEED_TOOLS_ENV]
|
|
124
|
-
end
|
|
125
|
-
config_hash
|
|
126
|
-
end
|
|
127
|
-
ActiveRecord::Base.configurations = database_config_hash
|
|
128
|
-
ActiveRecord::Base.establish_connection(database_config_hash)
|
|
129
|
-
ActiveRecord::Base.connection
|
|
130
|
-
rescue
|
|
131
|
-
end
|
|
132
|
-
end
|
|
133
|
-
# Verify that the necessary database tables are in place
|
|
134
|
-
# and if they're missing, create them
|
|
135
|
-
unless DatabaseFeedCache.table_exists?
|
|
136
|
-
DatabaseFeedCache.create_table
|
|
137
|
-
end
|
|
138
|
-
return nil
|
|
139
|
-
end
|
|
140
|
-
|
|
141
|
-
# Returns true if a connection to the database has been established and the
|
|
142
|
-
# required table structure is in place.
|
|
143
|
-
def DatabaseFeedCache.connected?
|
|
144
|
-
begin
|
|
145
|
-
ActiveRecord::Base.connection
|
|
146
|
-
return false if ActiveRecord::Base.configurations.nil?
|
|
147
|
-
return false unless DatabaseFeedCache.table_exists?
|
|
148
|
-
rescue => error
|
|
149
|
-
return false
|
|
150
|
-
end
|
|
151
|
-
return true
|
|
152
|
-
end
|
|
153
|
-
|
|
154
|
-
# True if the appropriate database table already exists
|
|
155
|
-
def DatabaseFeedCache.table_exists?
|
|
156
|
-
begin
|
|
157
|
-
ActiveRecord::Base.connection.execute "select id, url, title, " +
|
|
158
|
-
"link, xml_data, http_headers, last_retrieved " +
|
|
159
|
-
"from #{self.table_name()} limit 1"
|
|
160
|
-
rescue ActiveRecord::StatementInvalid
|
|
161
|
-
return false
|
|
162
|
-
rescue
|
|
163
|
-
return false
|
|
164
|
-
end
|
|
165
|
-
return true
|
|
166
|
-
end
|
|
167
|
-
|
|
168
|
-
# Creates the appropriate database table
|
|
169
|
-
def DatabaseFeedCache.create_table
|
|
170
|
-
unless DatabaseFeedCache.table_exists?
|
|
171
|
-
feeds_mysql = <<-SQL_END
|
|
172
|
-
CREATE TABLE `#{self.table_name()}` (
|
|
173
|
-
`id` int(10) unsigned NOT NULL auto_increment,
|
|
174
|
-
`url` varchar(255) default NULL,
|
|
175
|
-
`title` varchar(255) default NULL,
|
|
176
|
-
`link` varchar(255) default NULL,
|
|
177
|
-
`xml_data` longtext default NULL,
|
|
178
|
-
`http_headers` text default NULL,
|
|
179
|
-
`last_retrieved` datetime default NULL,
|
|
180
|
-
PRIMARY KEY (`id`)
|
|
181
|
-
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
|
|
182
|
-
SQL_END
|
|
183
|
-
feeds_sqlite = <<-SQL_END
|
|
184
|
-
CREATE TABLE '#{self.table_name()}' (
|
|
185
|
-
'id' INTEGER PRIMARY KEY NOT NULL,
|
|
186
|
-
'url' VARCHAR(255) DEFAULT NULL,
|
|
187
|
-
'title' VARCHAR(255) DEFAULT NULL,
|
|
188
|
-
'link' VARCHAR(255) DEFAULT NULL,
|
|
189
|
-
'xml_data' TEXT DEFAULT NULL,
|
|
190
|
-
'http_headers' TEXT DEFAULT NULL,
|
|
191
|
-
'last_retrieved' DATETIME DEFAULT NULL
|
|
192
|
-
);
|
|
193
|
-
SQL_END
|
|
194
|
-
feeds_psql = <<-SQL_END
|
|
195
|
-
CREATE TABLE #{self.table_name()} (
|
|
196
|
-
id SERIAL PRIMARY KEY NOT NULL,
|
|
197
|
-
url varchar(255) default NULL,
|
|
198
|
-
title varchar(255) default NULL,
|
|
199
|
-
link varchar(255) default NULL,
|
|
200
|
-
xml_data text default NULL,
|
|
201
|
-
http_headers text default NULL,
|
|
202
|
-
last_retrieved timestamp default NULL
|
|
203
|
-
);
|
|
204
|
-
SQL_END
|
|
205
|
-
table_creation_sql = nil
|
|
206
|
-
if configurations["adapter"] == "mysql"
|
|
207
|
-
table_creation_sql = feeds_mysql
|
|
208
|
-
elsif configurations["adapter"] == "sqlite"
|
|
209
|
-
table_creation_sql = feeds_sqlite
|
|
210
|
-
elsif configurations["adapter"] == "postgresql"
|
|
211
|
-
table_creation_sql = feeds_psql
|
|
212
|
-
end
|
|
213
|
-
if table_creation_sql.nil?
|
|
214
|
-
raise "Could not build #{self.table_name()} table."
|
|
215
|
-
else
|
|
216
|
-
connection.execute table_creation_sql
|
|
217
|
-
end
|
|
218
|
-
end
|
|
219
|
-
end
|
|
220
|
-
end
|
|
221
|
-
|
|
222
99
|
# Error raised when a feed cannot be retrieved
|
|
223
100
|
class FeedAccessError < StandardError
|
|
224
101
|
end
|
|
@@ -446,6 +323,8 @@ module FeedTools
|
|
|
446
323
|
'/usr/lib/tidy.so',
|
|
447
324
|
'C:\Program Files\Tidy\tidy.dll',
|
|
448
325
|
'C:\Tidy\tidy.dll',
|
|
326
|
+
'C:\Ruby\bin\tidy.dll',
|
|
327
|
+
'C:\Ruby\tidy.dll',
|
|
449
328
|
'/usr/local/lib',
|
|
450
329
|
'/opt/local/lib',
|
|
451
330
|
'/usr/lib'
|
|
@@ -558,6 +437,7 @@ module FeedTools
|
|
|
558
437
|
if (feed_uri.path =~ /^[\/]+/) == 0
|
|
559
438
|
feed_uri.path.gsub!(/^[\/]+/, "/")
|
|
560
439
|
end
|
|
440
|
+
feed_uri.host.downcase!
|
|
561
441
|
return feed_uri.to_s
|
|
562
442
|
rescue URI::InvalidURIError
|
|
563
443
|
return normalized_url
|
|
@@ -586,18 +466,14 @@ module FeedTools
|
|
|
586
466
|
end
|
|
587
467
|
|
|
588
468
|
# Converts a url into a urn:uuid: uri
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
# require 'uuidtools'
|
|
598
|
-
# tag_uri = UUID.sha1_create(normalized_url).to_uri_string
|
|
599
|
-
# return tag_uri
|
|
600
|
-
# end
|
|
469
|
+
def FeedTools.build_urn_uri(url)
|
|
470
|
+
unless url.kind_of? String
|
|
471
|
+
raise ArgumentError, "Expected String, got #{url.class.name}"
|
|
472
|
+
end
|
|
473
|
+
normalized_url = normalize_url(url)
|
|
474
|
+
require 'uuidtools'
|
|
475
|
+
return UUID.sha1_create(normalized_url).to_uri_string
|
|
476
|
+
end
|
|
601
477
|
|
|
602
478
|
# Returns true if the parameter appears to be a valid uri
|
|
603
479
|
def FeedTools.is_uri?(url)
|
|
@@ -896,10 +772,10 @@ module FeedTools
|
|
|
896
772
|
@http_headers = YAML.load(self.cache_object.http_headers)
|
|
897
773
|
@http_headers = {} unless @http_headers.kind_of? Hash
|
|
898
774
|
end
|
|
899
|
-
if
|
|
900
|
-
load_remote_feed!
|
|
901
|
-
else
|
|
775
|
+
if FeedTools.cache_only? || self.expired? == false
|
|
902
776
|
@live = false
|
|
777
|
+
else
|
|
778
|
+
load_remote_feed!
|
|
903
779
|
end
|
|
904
780
|
end
|
|
905
781
|
|
|
@@ -955,8 +831,8 @@ module FeedTools
|
|
|
955
831
|
FeedTools.user_agent unless FeedTools.user_agent.nil?
|
|
956
832
|
|
|
957
833
|
# The http feed access method
|
|
958
|
-
|
|
959
|
-
response_chain
|
|
834
|
+
http_fetch = lambda do |feed_url, http_headers, redirect_limit,
|
|
835
|
+
response_chain|
|
|
960
836
|
raise FeedAccessError, 'Redirect too deep' if redirect_limit == 0
|
|
961
837
|
feed_uri = nil
|
|
962
838
|
begin
|
|
@@ -995,7 +871,7 @@ module FeedTools
|
|
|
995
871
|
break
|
|
996
872
|
end
|
|
997
873
|
end
|
|
998
|
-
|
|
874
|
+
response
|
|
999
875
|
when Net::HTTPRedirection
|
|
1000
876
|
if response.code.to_i == 304
|
|
1001
877
|
response.error!
|
|
@@ -1012,7 +888,7 @@ module FeedTools
|
|
|
1012
888
|
# TODO: deal with stupid people using relative urls
|
|
1013
889
|
# in Location header
|
|
1014
890
|
# =================================================
|
|
1015
|
-
http_fetch(new_location, http_headers,
|
|
891
|
+
http_fetch.call(new_location, http_headers,
|
|
1016
892
|
redirect_limit - 1, response_chain)
|
|
1017
893
|
end
|
|
1018
894
|
else
|
|
@@ -1022,7 +898,7 @@ module FeedTools
|
|
|
1022
898
|
end
|
|
1023
899
|
|
|
1024
900
|
begin
|
|
1025
|
-
@http_response = http_fetch(self.url, headers)
|
|
901
|
+
@http_response = http_fetch.call(self.url, headers, 10, [])
|
|
1026
902
|
@http_headers = {}
|
|
1027
903
|
self.http_response.each_header do |header|
|
|
1028
904
|
self.http_headers[header.first.downcase] = header.last
|
|
@@ -1142,6 +1018,9 @@ module FeedTools
|
|
|
1142
1018
|
def xml
|
|
1143
1019
|
if @xml_doc.nil?
|
|
1144
1020
|
begin
|
|
1021
|
+
# TODO: :ignore_whitespace_nodes => :all
|
|
1022
|
+
# Add that?
|
|
1023
|
+
# ======================================
|
|
1145
1024
|
@xml_doc = Document.new(xml_data)
|
|
1146
1025
|
rescue
|
|
1147
1026
|
# Something failed, attempt to repair the xml with htree.
|
|
@@ -2446,9 +2325,17 @@ module FeedTools
|
|
|
2446
2325
|
elsif feed_type == "atom" && (version == nil || version == 0.0)
|
|
2447
2326
|
version = 0.3
|
|
2448
2327
|
end
|
|
2449
|
-
if feed_type == "rss" && (version == 0.9 || version == 1.0 ||
|
|
2328
|
+
if feed_type == "rss" && (version == 0.9 || version == 1.0 ||
|
|
2329
|
+
version == 1.1)
|
|
2450
2330
|
# RDF-based rss format
|
|
2451
|
-
return xml_builder.tag!("rdf:RDF"
|
|
2331
|
+
return xml_builder.tag!("rdf:RDF",
|
|
2332
|
+
"xmlns" => "http://purl.org/rss/1.0/",
|
|
2333
|
+
"xmlns:rdf" => "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
|
2334
|
+
"xmlns:dc" => "http://purl.org/dc/elements/1.1/",
|
|
2335
|
+
"xmlns:syn" => "http://purl.org/rss/1.0/modules/syndication/",
|
|
2336
|
+
"xmlns:taxo" => "http://purl.org/rss/1.0/modules/taxonomy/",
|
|
2337
|
+
"xmlns:itunes" => "http://www.itunes.com/DTDs/Podcast-1.0.dtd",
|
|
2338
|
+
"xmlns:media" => "http://search.yahoo.com/mrss") do
|
|
2452
2339
|
channel_attributes = {}
|
|
2453
2340
|
unless self.link.nil?
|
|
2454
2341
|
channel_attributes["rdf:about"] = CGI.escapeHTML(self.link)
|
|
@@ -2530,22 +2417,32 @@ module FeedTools
|
|
|
2530
2417
|
end
|
|
2531
2418
|
elsif feed_type == "rss"
|
|
2532
2419
|
# normal rss format
|
|
2533
|
-
return xml_builder.rss("version" =>
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2420
|
+
return xml_builder.rss("version" => "2.0",
|
|
2421
|
+
"xmlns:rdf" => "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
|
2422
|
+
"xmlns:dc" => "http://purl.org/dc/elements/1.1/",
|
|
2423
|
+
"xmlns:taxo" => "http://purl.org/rss/1.0/modules/taxonomy/",
|
|
2424
|
+
"xmlns:trackback" =>
|
|
2425
|
+
"http://madskills.com/public/xml/rss/module/trackback/",
|
|
2426
|
+
"xmlns:itunes" => "http://www.itunes.com/DTDs/Podcast-1.0.dtd",
|
|
2427
|
+
"xmlns:media" => "http://search.yahoo.com/mrss") do
|
|
2428
|
+
xml_builder.channel do
|
|
2429
|
+
unless title.nil? || title == ""
|
|
2430
|
+
xml_builder.title(title)
|
|
2431
|
+
end
|
|
2432
|
+
unless link.nil? || link == ""
|
|
2433
|
+
xml_builder.link(link)
|
|
2434
|
+
end
|
|
2435
|
+
unless description.nil? || description == ""
|
|
2436
|
+
xml_builder.description(description)
|
|
2437
|
+
end
|
|
2438
|
+
xml_builder.ttl((time_to_live / 1.minute).to_s)
|
|
2439
|
+
xml_builder.generator(
|
|
2440
|
+
"http://www.sporkmonger.com/projects/feedtools")
|
|
2441
|
+
build_xml_hook(feed_type, version, xml_builder)
|
|
2442
|
+
unless items.nil?
|
|
2443
|
+
for item in items
|
|
2444
|
+
item.build_xml(feed_type, version, xml_builder)
|
|
2445
|
+
end
|
|
2549
2446
|
end
|
|
2550
2447
|
end
|
|
2551
2448
|
end
|
|
@@ -2647,16 +2544,16 @@ module FeedTools
|
|
|
2647
2544
|
"http://www.sporkmonger.com/projects/feedtools")
|
|
2648
2545
|
if self.id != nil
|
|
2649
2546
|
unless FeedTools.is_uri? self.id
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2547
|
+
if self.link != nil
|
|
2548
|
+
xml_builder.id(FeedTools.build_urn_uri(self.link))
|
|
2549
|
+
else
|
|
2653
2550
|
raise "The unique id must be a valid URI."
|
|
2654
|
-
|
|
2551
|
+
end
|
|
2655
2552
|
else
|
|
2656
2553
|
xml_builder.id(self.id)
|
|
2657
2554
|
end
|
|
2658
|
-
|
|
2659
|
-
|
|
2555
|
+
elsif self.link != nil
|
|
2556
|
+
xml_builder.id(FeedTools.build_urn_uri(self.link))
|
|
2660
2557
|
else
|
|
2661
2558
|
raise "Cannot build feed, missing feed unique id."
|
|
2662
2559
|
end
|
|
@@ -2880,6 +2777,9 @@ module FeedTools
|
|
|
2880
2777
|
# Returns a REXML Document of the xml_data
|
|
2881
2778
|
def xml
|
|
2882
2779
|
if @xml_doc.nil?
|
|
2780
|
+
# TODO: :ignore_whitespace_nodes => :all
|
|
2781
|
+
# Add that?
|
|
2782
|
+
# ======================================
|
|
2883
2783
|
@xml_doc = Document.new(xml_data)
|
|
2884
2784
|
end
|
|
2885
2785
|
return @xml_doc
|
|
@@ -2984,10 +2884,7 @@ module FeedTools
|
|
|
2984
2884
|
if @description.nil?
|
|
2985
2885
|
unless root_node.nil?
|
|
2986
2886
|
repair_entities = false
|
|
2987
|
-
description_node = XPath.first(root_node, "
|
|
2988
|
-
if description_node.nil?
|
|
2989
|
-
description_node = XPath.first(root_node, "content:encoded")
|
|
2990
|
-
end
|
|
2887
|
+
description_node = XPath.first(root_node, "content:encoded")
|
|
2991
2888
|
if description_node.nil?
|
|
2992
2889
|
description_node = XPath.first(root_node, "content")
|
|
2993
2890
|
end
|
|
@@ -3000,6 +2897,9 @@ module FeedTools
|
|
|
3000
2897
|
if description_node.nil?
|
|
3001
2898
|
description_node = XPath.first(root_node, "body")
|
|
3002
2899
|
end
|
|
2900
|
+
if description_node.nil?
|
|
2901
|
+
description_node = XPath.first(root_node, "description")
|
|
2902
|
+
end
|
|
3003
2903
|
if description_node.nil?
|
|
3004
2904
|
description_node = XPath.first(root_node, "tagline")
|
|
3005
2905
|
end
|
|
@@ -4178,7 +4078,7 @@ module FeedTools
|
|
|
4178
4078
|
xml_builder.tag!("dc:date", time.iso8601)
|
|
4179
4079
|
end
|
|
4180
4080
|
unless tags.nil? || tags.size == 0
|
|
4181
|
-
xml_builder.tag!("
|
|
4081
|
+
xml_builder.tag!("taxo:topics") do
|
|
4182
4082
|
xml_builder.tag!("rdf:Bag") do
|
|
4183
4083
|
for tag in tags
|
|
4184
4084
|
xml_builder.tag!("rdf:li", tag)
|
|
@@ -4205,7 +4105,7 @@ module FeedTools
|
|
|
4205
4105
|
xml_builder.pubDate(time.rfc822)
|
|
4206
4106
|
end
|
|
4207
4107
|
unless tags.nil? || tags.size == 0
|
|
4208
|
-
xml_builder.tag!("
|
|
4108
|
+
xml_builder.tag!("taxo:topics") do
|
|
4209
4109
|
xml_builder.tag!("rdf:Bag") do
|
|
4210
4110
|
for tag in tags
|
|
4211
4111
|
xml_builder.tag!("rdf:li", tag)
|
data/rakefile
CHANGED
|
@@ -7,7 +7,7 @@ require 'rake/gempackagetask'
|
|
|
7
7
|
require 'rake/contrib/rubyforgepublisher'
|
|
8
8
|
|
|
9
9
|
PKG_NAME = 'feedtools'
|
|
10
|
-
PKG_VERSION = '0.2.
|
|
10
|
+
PKG_VERSION = '0.2.9'
|
|
11
11
|
PKG_FILE_NAME = "#{PKG_NAME}-#{PKG_VERSION}"
|
|
12
12
|
|
|
13
13
|
RELEASE_NAME = "REL #{PKG_VERSION}"
|
|
@@ -59,6 +59,7 @@ spec = Gem::Specification.new do |s|
|
|
|
59
59
|
end
|
|
60
60
|
|
|
61
61
|
s.add_dependency('activerecord', '>= 1.10.1')
|
|
62
|
+
s.add_dependency('uuidtools', '>= 0.1.1')
|
|
62
63
|
|
|
63
64
|
s.require_path = 'lib'
|
|
64
65
|
s.autorequire = 'feed_tools'
|
metadata
CHANGED
|
@@ -3,8 +3,8 @@ rubygems_version: 0.8.11
|
|
|
3
3
|
specification_version: 1
|
|
4
4
|
name: feedtools
|
|
5
5
|
version: !ruby/object:Gem::Version
|
|
6
|
-
version: 0.2.
|
|
7
|
-
date: 2005-09-
|
|
6
|
+
version: 0.2.9
|
|
7
|
+
date: 2005-09-25 00:00:00 -04:00
|
|
8
8
|
summary: "Parsing, generation, and caching system for xml news feeds."
|
|
9
9
|
require_paths:
|
|
10
10
|
- lib
|
|
@@ -33,6 +33,7 @@ files:
|
|
|
33
33
|
- install.rb
|
|
34
34
|
- README
|
|
35
35
|
- CHANGELOG
|
|
36
|
+
- lib/database_feed_cache.rb
|
|
36
37
|
- lib/feed_tools
|
|
37
38
|
- lib/feed_tools.rb
|
|
38
39
|
- lib/feed_tools/vendor
|
|
@@ -96,4 +97,14 @@ dependencies:
|
|
|
96
97
|
- ">="
|
|
97
98
|
- !ruby/object:Gem::Version
|
|
98
99
|
version: 1.10.1
|
|
100
|
+
version:
|
|
101
|
+
- !ruby/object:Gem::Dependency
|
|
102
|
+
name: uuidtools
|
|
103
|
+
version_requirement:
|
|
104
|
+
version_requirements: !ruby/object:Gem::Version::Requirement
|
|
105
|
+
requirements:
|
|
106
|
+
-
|
|
107
|
+
- ">="
|
|
108
|
+
- !ruby/object:Gem::Version
|
|
109
|
+
version: 0.1.1
|
|
99
110
|
version:
|