feed2email 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +21 -10
- data/README.md +138 -86
- data/bin/f2e +6 -4
- data/bin/feed2email +6 -4
- data/bin/feed2email-migrate +23 -0
- data/lib/feed2email.rb +6 -9
- data/lib/feed2email/cli.rb +112 -75
- data/lib/feed2email/config.rb +72 -52
- data/lib/feed2email/core_ext.rb +10 -0
- data/lib/feed2email/database.rb +75 -0
- data/lib/feed2email/entry.rb +149 -15
- data/lib/feed2email/feed.rb +111 -130
- data/lib/feed2email/feed_autodiscoverer.rb +8 -10
- data/lib/feed2email/migrate/convert_feeds_migration.rb +29 -0
- data/lib/feed2email/migrate/feeds_import_migration.rb +42 -0
- data/lib/feed2email/migrate/history_import_migration.rb +42 -0
- data/lib/feed2email/migrate/migration.rb +42 -0
- data/lib/feed2email/migrate/split_history_migration.rb +32 -0
- data/lib/feed2email/open-uri.rb +7 -0
- data/lib/feed2email/opml_exporter.rb +109 -0
- data/lib/feed2email/opml_importer.rb +52 -0
- data/lib/feed2email/redirection_checker.rb +2 -2
- data/lib/feed2email/smtp_connection.rb +59 -0
- data/lib/feed2email/version.rb +1 -1
- metadata +55 -30
- data/bin/feed2email-migrate-feedlist +0 -36
- data/bin/feed2email-migrate-history +0 -29
- data/lib/feed2email/feed_history.rb +0 -82
- data/lib/feed2email/feed_list.rb +0 -147
- data/lib/feed2email/lazy_smtp_connection.rb +0 -35
- data/lib/feed2email/mail.rb +0 -84
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'nokogiri'
|
2
|
-
require 'open-uri'
|
3
2
|
require 'uri'
|
3
|
+
require 'feed2email/open-uri'
|
4
4
|
|
5
5
|
module Feed2Email
|
6
6
|
class FeedAutodiscoverer
|
@@ -8,17 +8,15 @@ module Feed2Email
|
|
8
8
|
@uri = uri
|
9
9
|
end
|
10
10
|
|
11
|
-
def content_type;
|
11
|
+
def content_type; handle.content_type end
|
12
12
|
|
13
13
|
def feeds
|
14
|
-
|
15
|
-
fetch
|
16
|
-
@feeds = discoverable? ? discover : []
|
14
|
+
@feeds ||= discoverable? ? discover : []
|
17
15
|
end
|
18
16
|
|
19
17
|
private
|
20
18
|
|
21
|
-
def data;
|
19
|
+
def data; handle.read end
|
22
20
|
|
23
21
|
def discover
|
24
22
|
head = Nokogiri::HTML(data).at_css('head')
|
@@ -26,7 +24,7 @@ module Feed2Email
|
|
26
24
|
if base = head.at_css('base[href]')
|
27
25
|
base_uri = base['href']
|
28
26
|
else
|
29
|
-
base_uri =
|
27
|
+
base_uri = handle.base_uri.to_s
|
30
28
|
end
|
31
29
|
|
32
30
|
head.css('link[rel=alternate]').select {|link|
|
@@ -43,11 +41,11 @@ module Feed2Email
|
|
43
41
|
end
|
44
42
|
|
45
43
|
def discoverable?
|
46
|
-
content_type == 'text/html'
|
44
|
+
handle.content_type == 'text/html'
|
47
45
|
end
|
48
46
|
|
49
|
-
def
|
50
|
-
@
|
47
|
+
def handle
|
48
|
+
@handle ||= open(uri)
|
51
49
|
end
|
52
50
|
|
53
51
|
def uri; @uri end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'feed2email/migrate/migration'
|
2
|
+
|
3
|
+
module Feed2Email
|
4
|
+
module Migrate
|
5
|
+
class ConvertFeedsMigration < Migration
|
6
|
+
private
|
7
|
+
|
8
|
+
def applicable?
|
9
|
+
super && valid_data?
|
10
|
+
end
|
11
|
+
|
12
|
+
def filename
|
13
|
+
'feeds.yml'
|
14
|
+
end
|
15
|
+
|
16
|
+
def migrate
|
17
|
+
open(path, 'w') {|f| f.write(to_yaml) }
|
18
|
+
end
|
19
|
+
|
20
|
+
def to_yaml
|
21
|
+
data.map {|uri| { uri: uri, enabled: true } }.to_yaml
|
22
|
+
end
|
23
|
+
|
24
|
+
def valid_data?
|
25
|
+
data.is_a?(Array) && data.all? {|d| d.is_a?(String) }
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'feed2email/feed'
|
2
|
+
require 'feed2email/migrate/migration'
|
3
|
+
|
4
|
+
module Feed2Email
|
5
|
+
module Migrate
|
6
|
+
class FeedsImportMigration < Migration
|
7
|
+
def apply
|
8
|
+
applicable? && migrate
|
9
|
+
end
|
10
|
+
|
11
|
+
private
|
12
|
+
|
13
|
+
def applicable?
|
14
|
+
super && table_empty? && valid_data?
|
15
|
+
end
|
16
|
+
|
17
|
+
def filename
|
18
|
+
'feeds.yml'
|
19
|
+
end
|
20
|
+
|
21
|
+
def migrate
|
22
|
+
data.each do |feed|
|
23
|
+
Feed.create(
|
24
|
+
uri: feed[:uri],
|
25
|
+
enabled: feed[:enabled],
|
26
|
+
etag: feed[:etag],
|
27
|
+
last_modified: feed[:last_modified],
|
28
|
+
last_processed_at: Time.now
|
29
|
+
)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def table_empty?
|
34
|
+
Feed.empty?
|
35
|
+
end
|
36
|
+
|
37
|
+
def valid_data?
|
38
|
+
data.is_a?(Array) && data.all? {|d| d.is_a?(Hash) && d.has_key?(:uri) }
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'digest/md5'
|
2
|
+
require 'feed2email/entry'
|
3
|
+
require 'feed2email/feed'
|
4
|
+
require 'feed2email/migrate/migration'
|
5
|
+
|
6
|
+
module Feed2Email
|
7
|
+
module Migrate
|
8
|
+
class HistoryImportMigration < Migration
|
9
|
+
def apply
|
10
|
+
applicable? && migrate
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
|
15
|
+
def applicable?
|
16
|
+
table_empty?
|
17
|
+
end
|
18
|
+
|
19
|
+
def feed_history_data(feed_uri)
|
20
|
+
YAML.load(open(feed_history_path(feed_uri)))
|
21
|
+
end
|
22
|
+
|
23
|
+
def feed_history_path(feed_uri)
|
24
|
+
root.join("history-#{Digest::MD5.hexdigest(feed_uri)}.yml")
|
25
|
+
end
|
26
|
+
|
27
|
+
def migrate
|
28
|
+
Feed.each do |feed|
|
29
|
+
if feed_history_path(feed.uri).exist?
|
30
|
+
feed_history_data(feed.uri).each do |entry_uri|
|
31
|
+
Entry.find_or_create(feed_id: feed.id, uri: entry_uri)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def table_empty?
|
38
|
+
Entry.empty?
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'yaml'
|
3
|
+
require 'feed2email'
|
4
|
+
|
5
|
+
module Feed2Email
|
6
|
+
module Migrate
|
7
|
+
class Migration
|
8
|
+
def apply
|
9
|
+
applicable? && backup_file && migrate
|
10
|
+
end
|
11
|
+
|
12
|
+
private
|
13
|
+
|
14
|
+
def applicable?
|
15
|
+
file_exists?
|
16
|
+
end
|
17
|
+
|
18
|
+
def backup_file
|
19
|
+
begin
|
20
|
+
FileUtils.cp(path, "#{path}.bak")
|
21
|
+
true
|
22
|
+
rescue
|
23
|
+
false
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def data
|
28
|
+
@data ||= YAML.load(open(path))
|
29
|
+
end
|
30
|
+
|
31
|
+
def file_exists?
|
32
|
+
path.exist?
|
33
|
+
end
|
34
|
+
|
35
|
+
def path
|
36
|
+
root.join(filename)
|
37
|
+
end
|
38
|
+
|
39
|
+
def root; Feed2Email.root end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'digest/md5'
|
2
|
+
require 'feed2email/migrate/migration'
|
3
|
+
|
4
|
+
module Feed2Email
|
5
|
+
module Migrate
|
6
|
+
class SplitHistoryMigration < Migration
|
7
|
+
private
|
8
|
+
|
9
|
+
def applicable?
|
10
|
+
super && pending?
|
11
|
+
end
|
12
|
+
|
13
|
+
def feed_history_path(feed_uri)
|
14
|
+
root.join("history-#{Digest::MD5.hexdigest(feed_uri)}.yml")
|
15
|
+
end
|
16
|
+
|
17
|
+
def filename
|
18
|
+
'history.yml'
|
19
|
+
end
|
20
|
+
|
21
|
+
def migrate
|
22
|
+
data.each do |feed_uri, entries|
|
23
|
+
open(feed_history_path(feed_uri), 'w') {|f| f.write(entries.to_yaml) }
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def pending?
|
28
|
+
Dir[root.join('history-*.yml')].empty?
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,109 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'nokogiri'
|
3
|
+
require 'uri'
|
4
|
+
|
5
|
+
module Feed2Email
|
6
|
+
class OPMLExporter
|
7
|
+
MAX_REDIRECTS = 10
|
8
|
+
|
9
|
+
def self.export(path)
|
10
|
+
require 'feed2email/feed'
|
11
|
+
|
12
|
+
open(path, 'w') do |f|
|
13
|
+
uris = Feed.by_smallest_id.select_map(:uri)
|
14
|
+
|
15
|
+
if new(uris).export(f) > 0
|
16
|
+
uris.size
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def initialize(uris)
|
22
|
+
@uris = uris
|
23
|
+
end
|
24
|
+
|
25
|
+
def export(io)
|
26
|
+
io.write(xml)
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def builder
|
32
|
+
Nokogiri::XML::Builder.new do |xml|
|
33
|
+
xml.root {
|
34
|
+
xml.opml(version: '2.0') {
|
35
|
+
xml.head {
|
36
|
+
xml.title 'feed2email subscriptions'
|
37
|
+
xml.dateCreated Time.now
|
38
|
+
xml.ownerName ENV['USER']
|
39
|
+
xml.docs 'http://dev.opml.org/spec2.html'
|
40
|
+
}
|
41
|
+
xml.body {
|
42
|
+
uris.each do |uri|
|
43
|
+
xml.outline(text: uri, type: feed_type(uri), xmlUrl: uri)
|
44
|
+
end
|
45
|
+
}
|
46
|
+
}
|
47
|
+
}
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
# Adjusted from
|
52
|
+
# https://github.com/yugui/rubycommitters/blob/master/opml-generator.rb
|
53
|
+
def feed_type(url)
|
54
|
+
uri = nil
|
55
|
+
redirects = 0
|
56
|
+
|
57
|
+
loop do
|
58
|
+
uri = URI.parse(url)
|
59
|
+
|
60
|
+
begin
|
61
|
+
response = Net::HTTP.start(uri.host, uri.port) {|http|
|
62
|
+
http.head(uri.request_uri)
|
63
|
+
}
|
64
|
+
rescue
|
65
|
+
break
|
66
|
+
end
|
67
|
+
|
68
|
+
if response.code =~ /\A3\d\d\z/
|
69
|
+
redirects += 1
|
70
|
+
return unless response['location'] && redirects <= MAX_REDIRECTS
|
71
|
+
url = response['location']
|
72
|
+
next
|
73
|
+
end
|
74
|
+
|
75
|
+
case response['content-type'][/[^;]+/]
|
76
|
+
when 'text/rss', 'text/rss+xml', 'application/rss+xml',
|
77
|
+
'application/rdf+xml', 'application/xml', 'text/xml'
|
78
|
+
return 'rss'
|
79
|
+
when 'text/atom', 'text/atom+xml', 'application/atom+xml'
|
80
|
+
return 'atom'
|
81
|
+
else
|
82
|
+
break
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
case File.extname(uri.path)
|
87
|
+
when '.rdf', '.rss'
|
88
|
+
return 'rss'
|
89
|
+
when '.atom'
|
90
|
+
return 'atom'
|
91
|
+
end
|
92
|
+
|
93
|
+
case File.basename(uri.path)
|
94
|
+
when 'rss.xml', 'rdf.xml'
|
95
|
+
return 'rss'
|
96
|
+
when 'atom.xml'
|
97
|
+
return 'atom'
|
98
|
+
else
|
99
|
+
return
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
def uris; @uris end
|
104
|
+
|
105
|
+
def xml
|
106
|
+
builder.to_xml
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'nokogiri'
|
2
|
+
|
3
|
+
module Feed2Email
|
4
|
+
class OPMLImporter
|
5
|
+
def self.import(path)
|
6
|
+
require 'feed2email/feed'
|
7
|
+
|
8
|
+
n = 0
|
9
|
+
|
10
|
+
open(path) do |f|
|
11
|
+
new(f).import do |uri|
|
12
|
+
if feed = Feed[uri: uri]
|
13
|
+
warn "Feed already exists: #{feed}"
|
14
|
+
else
|
15
|
+
feed = Feed.new(uri: uri)
|
16
|
+
|
17
|
+
if feed.save(raise_on_failure: false)
|
18
|
+
puts "Imported feed: #{feed}"
|
19
|
+
n += 1
|
20
|
+
else
|
21
|
+
warn "Failed to import feed: #{feed}"
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
n
|
28
|
+
end
|
29
|
+
|
30
|
+
def initialize(io)
|
31
|
+
@io = io
|
32
|
+
end
|
33
|
+
|
34
|
+
def import(&blk)
|
35
|
+
uris.each(&blk)
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
def data
|
41
|
+
io.read
|
42
|
+
end
|
43
|
+
|
44
|
+
def io; @io end
|
45
|
+
|
46
|
+
def uris
|
47
|
+
Nokogiri::XML(data).css('opml body outline').map {|outline|
|
48
|
+
outline['xmlUrl']
|
49
|
+
}.compact
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
require 'net/smtp'
|
2
|
+
require 'feed2email/configurable'
|
3
|
+
|
4
|
+
module Feed2Email
|
5
|
+
def self.smtp_connection
|
6
|
+
@smtp_connection
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.smtp_connection=(smtp_connection)
|
10
|
+
@smtp_connection = smtp_connection
|
11
|
+
end
|
12
|
+
|
13
|
+
class SMTPConnection
|
14
|
+
extend Configurable
|
15
|
+
|
16
|
+
def self.setup
|
17
|
+
Feed2Email.smtp_connection = new(
|
18
|
+
config.slice(*config.keys.grep(/\Asmtp_/))
|
19
|
+
)
|
20
|
+
at_exit { Feed2Email.smtp_connection.finish }
|
21
|
+
end
|
22
|
+
|
23
|
+
def initialize(options)
|
24
|
+
@options = options
|
25
|
+
end
|
26
|
+
|
27
|
+
def finish
|
28
|
+
smtp.finish if started?
|
29
|
+
end
|
30
|
+
|
31
|
+
def sendmail(*args, &block)
|
32
|
+
start unless started?
|
33
|
+
smtp.sendmail(*args, &block) # delegate
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def options; @options end
|
39
|
+
|
40
|
+
def smtp
|
41
|
+
return @smtp if @smtp
|
42
|
+
@smtp = Net::SMTP.new(options['smtp_host'], options['smtp_port'])
|
43
|
+
@smtp.enable_starttls if options['smtp_starttls']
|
44
|
+
@smtp
|
45
|
+
end
|
46
|
+
|
47
|
+
def start
|
48
|
+
smtp.start('localhost',
|
49
|
+
options['smtp_user'],
|
50
|
+
options['smtp_pass'],
|
51
|
+
options['smtp_auth'].to_sym
|
52
|
+
)
|
53
|
+
end
|
54
|
+
|
55
|
+
def started?
|
56
|
+
smtp.started? # delegate
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|