harvester 0.8.0.pre.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG.rdoc +45 -0
- data/README.rdoc +74 -0
- data/Rakefile +28 -0
- data/bin/harvester +13 -0
- data/bin/harvester-chart +5 -0
- data/bin/harvester-clock +35 -0
- data/bin/harvester-db +15 -0
- data/bin/harvester-fetch +5 -0
- data/bin/harvester-generate +5 -0
- data/bin/harvester-jabber +6 -0
- data/bin/harvester-new +25 -0
- data/bin/harvester-post +5 -0
- data/bin/harvester-run +14 -0
- data/collections.yaml +15 -0
- data/config.yaml +13 -0
- data/data/ent/HTMLlat1.ent +194 -0
- data/data/ent/HTMLspecial.ent +77 -0
- data/data/ent/HTMLsymbol.ent +241 -0
- data/data/sql/dbd-mysql-isotime.diff +11 -0
- data/data/sql/harvester-0.6-mysql.diff +59 -0
- data/data/sql/harvester-0.7-mysql.diff +39 -0
- data/data/sql/mysql/chart.sql +1 -0
- data/data/sql/mysql/create.table.enclosures.sql +9 -0
- data/data/sql/mysql/create.table.items.sql +8 -0
- data/data/sql/mysql/create.table.jabbersettings.sql +5 -0
- data/data/sql/mysql/create.table.jabbersubscriptions.sql +5 -0
- data/data/sql/mysql/create.table.sources.sql +9 -0
- data/data/sql/mysql/create.view.last48hours.sql +1 -0
- data/data/sql/postgresql/chart.sql +1 -0
- data/data/sql/postgresql/create.table.enclosures.sql +9 -0
- data/data/sql/postgresql/create.table.items.sql +8 -0
- data/data/sql/postgresql/create.table.jabbersettings.sql +5 -0
- data/data/sql/postgresql/create.table.jabbersubscriptions.sql +5 -0
- data/data/sql/postgresql/create.table.sources.sql +9 -0
- data/data/sql/postgresql/create.view.last48hours.sql +1 -0
- data/data/sql/sqlite3/chart.sql +1 -0
- data/data/sql/sqlite3/create.table.enclosures.sql +9 -0
- data/data/sql/sqlite3/create.table.items.sql +8 -0
- data/data/sql/sqlite3/create.table.jabbersettings.sql +5 -0
- data/data/sql/sqlite3/create.table.jabbersubscriptions.sql +5 -0
- data/data/sql/sqlite3/create.table.sources.sql +9 -0
- data/data/sql/sqlite3/create.view.last48hours.sql +1 -0
- data/data/templates/atom-all.xml +88 -0
- data/data/templates/atom.xml +88 -0
- data/data/templates/index.html +412 -0
- data/data/templates/rss-all.rdf +86 -0
- data/data/templates/rss.rdf +85 -0
- data/data/templates/static/harvester.css +365 -0
- data/data/templates/static/harvester.gif +0 -0
- data/data/templates/static/harvester_ie7.css +15 -0
- data/data/templates/static/harvester_lte_ie6.css +27 -0
- data/harvester.gemspec +35 -0
- data/lib/harvester.rb +132 -0
- data/lib/harvester/chart.rb +72 -0
- data/lib/harvester/db.rb +123 -0
- data/lib/harvester/fetch.rb +96 -0
- data/lib/harvester/generate.rb +152 -0
- data/lib/harvester/generator/entity_translator.rb +46 -0
- data/lib/harvester/generator/link_absolutizer.rb +39 -0
- data/lib/harvester/jabber.rb +443 -0
- data/lib/harvester/mrss.rb +355 -0
- data/lib/harvester/post.rb +19 -0
- metadata +237 -0
@@ -0,0 +1,152 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require_relative '../harvester'
|
4
|
+
require_relative 'generator/link_absolutizer'
|
5
|
+
require_relative 'generator/entity_translator'
|
6
|
+
|
7
|
+
require 'fileutils'
|
8
|
+
require 'time'
|
9
|
+
require 'rexml/document'
|
10
|
+
begin
|
11
|
+
require 'xml/xslt'
|
12
|
+
rescue LoadError
|
13
|
+
require 'xml/libxslt'
|
14
|
+
end
|
15
|
+
|
16
|
+
class Harvester
|
17
|
+
module GENERATE; end
|
18
|
+
|
19
|
+
# generates the static html/feed files
|
20
|
+
def generate!
|
21
|
+
info "GENERATE"
|
22
|
+
|
23
|
+
f = Generator.new @dbi, @logger
|
24
|
+
xslt = XML::XSLT.new
|
25
|
+
xslt.xml = f.generate_root.to_s
|
26
|
+
|
27
|
+
default_template_dir = File.dirname(__FILE__) + '/../../data/templates'
|
28
|
+
template_dir = @settings['templates'] || default_template_dir
|
29
|
+
output_dir = @settings['output']
|
30
|
+
|
31
|
+
task "copy static files" do
|
32
|
+
FileUtils.mkdir_p output_dir
|
33
|
+
FileUtils.cp_r Dir[File.join( template_dir, 'static', '*' )], output_dir
|
34
|
+
end
|
35
|
+
|
36
|
+
begin
|
37
|
+
Dir.foreach(template_dir) { |template_file|
|
38
|
+
next if template_file =~ /^\./ || template_file == 'static'
|
39
|
+
|
40
|
+
task "process #{template_file}" do
|
41
|
+
xslt.xsl = File.join( template_dir, template_file )
|
42
|
+
File::open( File.join( output_dir, template_file ), 'w') { |f| f.write(xslt.serve) }
|
43
|
+
end
|
44
|
+
}
|
45
|
+
rescue Errno::ENOENT
|
46
|
+
warn "Couldn't find templates directory, fallback to default templates!"
|
47
|
+
template_dir = default_template_dir
|
48
|
+
retry
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
# generates the static html/feed files
|
54
|
+
class Harvester::Generator
|
55
|
+
FUNC_NAMESPACE = 'http://astroblog.spaceboyz.net/harvester/xslt-functions'
|
56
|
+
|
57
|
+
def initialize(dbi, logger)
|
58
|
+
@dbi = dbi
|
59
|
+
@logger = logger
|
60
|
+
%w(collection-items feed-items item-description item-images item-enclosures).each { |func|
|
61
|
+
XML::XSLT.extFunction(func, FUNC_NAMESPACE, self)
|
62
|
+
}
|
63
|
+
end
|
64
|
+
|
65
|
+
def generate_root
|
66
|
+
root = REXML::Element.new('collections')
|
67
|
+
@dbi.execute("SELECT collection FROM sources GROUP BY collection").each{ |name,|
|
68
|
+
collection = root.add(REXML::Element.new('collection'))
|
69
|
+
collection.attributes['name'] = name
|
70
|
+
@dbi.execute("SELECT rss,title,link,description FROM sources WHERE collection=?", name).each{ |rss,title,link,description|
|
71
|
+
#p [title, description]
|
72
|
+
feed = collection.add(REXML::Element.new('feed'))
|
73
|
+
feed.add(REXML::Element.new('rss')).text = rss
|
74
|
+
feed.add(REXML::Element.new('title')).text = title
|
75
|
+
feed.add(REXML::Element.new('link')).text = link
|
76
|
+
feed.add(REXML::Element.new('description')).text = description
|
77
|
+
}
|
78
|
+
}
|
79
|
+
|
80
|
+
EntityTranslator.run(root, true, @logger)
|
81
|
+
end
|
82
|
+
|
83
|
+
def collection_items(collection, max=23)
|
84
|
+
items = REXML::Element.new('items')
|
85
|
+
@dbi.execute("SELECT items.title,items.date,items.link,items.rss FROM items,sources WHERE items.rss=sources.rss AND sources.collection LIKE ? ORDER BY items.date DESC LIMIT ?", collection, max.to_i).each{ |title,date,link,rss|
|
86
|
+
if title # TODO: debug (sqlite)
|
87
|
+
item = items.add(REXML::Element.new('item'))
|
88
|
+
item.add(REXML::Element.new('title')).text = title
|
89
|
+
item.add(REXML::Element.new('date')).text = Time.parse(date).xmlschema
|
90
|
+
item.add(REXML::Element.new('link')).text = link
|
91
|
+
item.add(REXML::Element.new('rss')).text = rss
|
92
|
+
end
|
93
|
+
}
|
94
|
+
|
95
|
+
EntityTranslator.run(items, true, @logger)
|
96
|
+
end
|
97
|
+
|
98
|
+
def feed_items(rss, max=23)
|
99
|
+
items = REXML::Element.new('items')
|
100
|
+
@dbi.execute("SELECT title,date,link FROM items WHERE rss=? ORDER BY date DESC LIMIT ?", rss, max.to_i).each{ |title,date,link| #p rss,title,date,link
|
101
|
+
# p title
|
102
|
+
if title # TODO: debug (sqlite)
|
103
|
+
item = items.add(REXML::Element.new('item'))
|
104
|
+
item.add(REXML::Element.new('title')).text = title
|
105
|
+
item.add(REXML::Element.new('date')).text = Time.parse(date).xmlschema
|
106
|
+
item.add(REXML::Element.new('link')).text = link
|
107
|
+
end
|
108
|
+
}
|
109
|
+
|
110
|
+
EntityTranslator.run(items, true, @logger)
|
111
|
+
end
|
112
|
+
|
113
|
+
def item_description(rss, item_link)
|
114
|
+
# FIXME!!!! tmp ugly sqlite fix
|
115
|
+
if @dbi.driver.class.to_s =~ /sqlite3/i
|
116
|
+
a= "SELECT description FROM items WHERE rss='%s' AND link='%s'" % [rss, item_link].map{|e|::SQLite3::Database.quote(e) }
|
117
|
+
b= @dbi.execute(a).fetch
|
118
|
+
else
|
119
|
+
b= @dbi.execute("SELECT description FROM items WHERE rss=? AND link=?", rss, item_link).fetch
|
120
|
+
end
|
121
|
+
b.each{ |desc,|
|
122
|
+
desc = EntityTranslator.run(desc, false, @logger)
|
123
|
+
desc = LinkAbsolutizer.run(desc, item_link, @logger)
|
124
|
+
return desc
|
125
|
+
}
|
126
|
+
''
|
127
|
+
end
|
128
|
+
|
129
|
+
def item_images(rss, item_link)
|
130
|
+
desc = "<description>" + item_description(rss, item_link) + "</description>"
|
131
|
+
images = REXML::Element.new('images')
|
132
|
+
REXML::Document.new(desc.to_s).root.each_element('//img') { |img|
|
133
|
+
images.add img
|
134
|
+
}
|
135
|
+
mages
|
136
|
+
end
|
137
|
+
|
138
|
+
def item_enclosures(rss, link)
|
139
|
+
#p [rss,link]
|
140
|
+
enclosures = REXML::Element.new('enclosures')
|
141
|
+
@dbi.execute("SELECT href, mime, title, length FROM enclosures WHERE rss=? AND link=? ORDER BY length DESC", rss, link).each{ |href,mime,title,length|
|
142
|
+
enclosure = enclosures.add(REXML::Element.new('enclosure'))
|
143
|
+
enclosure.add(REXML::Element.new('href')).text = href
|
144
|
+
enclosure.add(REXML::Element.new('mime')).text = mime
|
145
|
+
enclosure.add(REXML::Element.new('title')).text = title
|
146
|
+
enclosure.add(REXML::Element.new('length')).text = length
|
147
|
+
}
|
148
|
+
#p enclosures.to_s
|
149
|
+
enclosures
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
class Harvester; class Generator; end; end
|
3
|
+
|
4
|
+
# This module translates old-fashioned entities into utf-8
|
5
|
+
class Harvester::Generator::EntityTranslator
|
6
|
+
def self.run(doc, with_xmldecl = true, logger = nil)
|
7
|
+
logger ||= Logger.new(STDOUT)
|
8
|
+
|
9
|
+
@entities = {}
|
10
|
+
%w(HTMLlat1.ent HTMLsymbol.ent HTMLspecial.ent).each do |file|
|
11
|
+
begin
|
12
|
+
load_entities_from_file(
|
13
|
+
File.expand_path( File.dirname(__FILE__) + '/../../../data/ent/' + file )
|
14
|
+
)
|
15
|
+
#rescue Errno::ENOENT
|
16
|
+
# system("wget http://www.w3.org/TR/html4/#{file}")
|
17
|
+
# load_entities_from_file(file)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
translate_entities(doc, with_xmldecl)
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.load_entities_from_file(filename)
|
24
|
+
File.read(filename).scan(/<!ENTITY +(.+?) +CDATA +"(.+?)".+?>/m) do |ent,code|
|
25
|
+
@entities[ent] = code
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.translate_entities(doc, with_xmldecl = true)
|
30
|
+
oldclass = doc.class
|
31
|
+
doc = doc.to_s
|
32
|
+
|
33
|
+
@entities.each do |ent,code|
|
34
|
+
doc.gsub!("&#{ent};", code)
|
35
|
+
end
|
36
|
+
|
37
|
+
doc = "<?xml version='1.0' encoding='utf-8'?>\n#{doc}" if with_xmldecl
|
38
|
+
|
39
|
+
if oldclass == REXML::Element
|
40
|
+
REXML::Document.new(doc).root
|
41
|
+
else
|
42
|
+
doc
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
class Harvester; class Generator; end; end
|
3
|
+
|
4
|
+
# This module rewrites relative to absolute links
|
5
|
+
module Harvester::Generator::LinkAbsolutizer
|
6
|
+
def self.run(body, base, logger = nil)
|
7
|
+
logger ||= Logger.new(STDOUT)
|
8
|
+
require 'hpricot'
|
9
|
+
|
10
|
+
html = Hpricot("<html><body>#{body}</body></html>")
|
11
|
+
(html/'a').each { |a|
|
12
|
+
begin
|
13
|
+
f = a.get_attribute('href')
|
14
|
+
t = URI::join(base, f.to_s).to_s
|
15
|
+
logger.debug "* rewriting #{f.inspect} => #{t.inspect}" if f != t
|
16
|
+
a.set_attribute('href', t)
|
17
|
+
rescue URI::Error
|
18
|
+
logger.debug "* cannot rewrite relative URL: #{a.get_attribute('href').inspect}" unless a.get_attribute('href') =~ /^[a-z]{2,10}:/
|
19
|
+
end
|
20
|
+
}
|
21
|
+
(html/'img').each { |img|
|
22
|
+
begin
|
23
|
+
f = img.get_attribute('src')
|
24
|
+
t = URI::join(base, f.to_s).to_s
|
25
|
+
logger.debug "* rewriting #{f.inspect} => #{t.inspect}" if f != t
|
26
|
+
img.set_attribute('src', t)
|
27
|
+
rescue URI::Error
|
28
|
+
logger.debug "* cannot rewrite relative URL: #{img.get_attribute('href').inspect}" unless img.get_attribute('href') =~ /^[a-z]{2,10}:/
|
29
|
+
end
|
30
|
+
}
|
31
|
+
html.search('/html/body/*').to_s
|
32
|
+
rescue Hpricot::Error => e
|
33
|
+
logger.error "Hpricot::Error: #{e}"
|
34
|
+
body
|
35
|
+
rescue LoadError
|
36
|
+
logger.warn "* hpricot not found, will not mangle relative links in <description/>"
|
37
|
+
body
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,443 @@
|
|
1
|
+
# encoding: ascii
|
2
|
+
|
3
|
+
require_relative '../harvester'
|
4
|
+
|
5
|
+
#require 'fastthread'
|
6
|
+
require 'xmpp4r'
|
7
|
+
require 'xmpp4r/discovery'
|
8
|
+
require 'xmpp4r/version'
|
9
|
+
require 'xmpp4r/roster'
|
10
|
+
require 'xmpp4r/dataforms'
|
11
|
+
require 'xmpp4r/vcard'
|
12
|
+
|
13
|
+
TABLE_SUBSCRIPTIONS = 'jabbersubscriptions'
|
14
|
+
TABLE_SETTINGS = 'jabbersettings'
|
15
|
+
|
16
|
+
Jabber::debug = true
|
17
|
+
|
18
|
+
class ChatState
|
19
|
+
def initialize(question, &block)
|
20
|
+
@question = question
|
21
|
+
@block = block
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
class ChatDialog
|
26
|
+
def initialize(&block)
|
27
|
+
@sendblock = block
|
28
|
+
@finished = false
|
29
|
+
@state = nil
|
30
|
+
end
|
31
|
+
def set_state(question, &state)
|
32
|
+
send question
|
33
|
+
@state = state
|
34
|
+
end
|
35
|
+
def send(str)
|
36
|
+
@sendblock.call str
|
37
|
+
end
|
38
|
+
def finished?
|
39
|
+
@finished
|
40
|
+
end
|
41
|
+
def finish!
|
42
|
+
@finished = true
|
43
|
+
end
|
44
|
+
def on_message(msg)
|
45
|
+
@state.call msg
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
class Interview < ChatDialog
|
50
|
+
def initialize(dbi, user, collections, &block)
|
51
|
+
raise 'No collections found!' unless collections.size > 0
|
52
|
+
|
53
|
+
super(&block)
|
54
|
+
@collections = collections
|
55
|
+
@collections_keys = collections.keys
|
56
|
+
|
57
|
+
set_state("Hello, I'm the Harvester Jabber service, aka NotAstroBot. " +
|
58
|
+
"Type \"start\" to subscribe to feeds selectively.") { |msg|
|
59
|
+
if msg == 'start'
|
60
|
+
|
61
|
+
set_state("Should I respect your online status by sending you notifications only when you're online? Please notice that you need to grant authorization to receive presence updates from you in that case.") { |msg|
|
62
|
+
if msg == 'yes' or msg == 'no'
|
63
|
+
respect_status = (msg == 'yes')
|
64
|
+
|
65
|
+
set_state("What type of message may I send to you? Valid answers are \"normal\", \"headline\" and \"chat\".") { |msg|
|
66
|
+
if msg == 'normal' or msg == 'headline' or msg == 'chat'
|
67
|
+
dbi.do "DELETE FROM #{TABLE_SETTINGS} WHERE JID=?", user
|
68
|
+
dbi.do "INSERT INTO #{TABLE_SETTINGS} (jid, respect_status, message_type) VALUES (?, ?, ?)",
|
69
|
+
user, respect_status, msg
|
70
|
+
|
71
|
+
collections_i = 0
|
72
|
+
|
73
|
+
set_state(collection_question(collections_i)) { |msg|
|
74
|
+
if msg == 'yes' or msg == 'no'
|
75
|
+
puts "#{@collections_keys[collections_i]}: #{msg}"
|
76
|
+
dbi.execute "DELETE FROM #{TABLE_SUBSCRIPTIONS} WHERE jid=? AND collection=?", user, @collections_keys[collections_i]
|
77
|
+
if msg == 'yes'
|
78
|
+
dbi.do "INSERT INTO #{TABLE_SUBSCRIPTIONS} (jid, collection) VALUES (?, ?)", user, @collections_keys[collections_i]
|
79
|
+
end
|
80
|
+
|
81
|
+
collections_i += 1
|
82
|
+
if collections_i < @collections.size
|
83
|
+
send collection_question(collections_i)
|
84
|
+
else
|
85
|
+
finish!
|
86
|
+
set_state('We\'ve done this interview. Talk to me if you want to repeat.') { |msg|
|
87
|
+
}
|
88
|
+
end
|
89
|
+
else
|
90
|
+
send 'I don\'t understand you. Please reply with either "yes" or "no".'
|
91
|
+
end
|
92
|
+
}
|
93
|
+
end
|
94
|
+
}
|
95
|
+
end
|
96
|
+
}
|
97
|
+
end
|
98
|
+
}
|
99
|
+
end
|
100
|
+
|
101
|
+
def collection_question(i)
|
102
|
+
if i >= @collections.size
|
103
|
+
nil
|
104
|
+
else
|
105
|
+
"Do you want to receive updates to the collection \"#{@collections_keys[i]}\", which include " +
|
106
|
+
@collections[@collections_keys[i]].collect { |rss,title|
|
107
|
+
title
|
108
|
+
}.join(', ') + '? ("yes" or "no")'
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
|
114
|
+
def duration_to_s(duration)
|
115
|
+
d = duration.to_i
|
116
|
+
r = []
|
117
|
+
while d >= 24 * 60 * 60
|
118
|
+
r << "#{d / (24 * 60 * 60)} days"
|
119
|
+
d %= 24 * 60 * 60
|
120
|
+
end
|
121
|
+
while d >= 60 * 60
|
122
|
+
r << "#{d / (60 * 60)} hrs"
|
123
|
+
d %= 60 * 60
|
124
|
+
end
|
125
|
+
while d >= 60
|
126
|
+
r << "#{d / 60} min"
|
127
|
+
d %= 60
|
128
|
+
end
|
129
|
+
(r.size > 0) ? r.join(', ') : 'no time'
|
130
|
+
end
|
131
|
+
|
132
|
+
|
133
|
+
class Harvester
|
134
|
+
def jabber!
|
135
|
+
warn "The jabber bot is not supported, yet. To nevertheless use it, remove this code line."; exit
|
136
|
+
|
137
|
+
collections = {}
|
138
|
+
|
139
|
+
dbi = @dbi
|
140
|
+
config = @config
|
141
|
+
|
142
|
+
cl = Jabber::Client.new Jabber::JID.new(config['jabber']['jid'])
|
143
|
+
cl.on_exception { |e,|
|
144
|
+
puts "HICKUP: #{e.class}: #{e}\n#{e.backtrace.join("\n")}"
|
145
|
+
begin
|
146
|
+
sleep 5
|
147
|
+
cl.connect config['jabber']['host'] || 'localhost'
|
148
|
+
cl.auth config['jabber']['password']
|
149
|
+
rescue
|
150
|
+
sleep 10
|
151
|
+
retry
|
152
|
+
end
|
153
|
+
}
|
154
|
+
cl.connect config['jabber']['host'] || 'localhost'
|
155
|
+
cl.auth config['jabber']['password']
|
156
|
+
|
157
|
+
Jabber::Version::SimpleResponder.new(cl, 'Harvester', '0.6', IO.popen('uname -sr') { |io| io.readlines.to_s.strip })
|
158
|
+
|
159
|
+
roster = Jabber::Roster::Helper.new(cl)
|
160
|
+
roster.add_subscription_request_callback { |item,presence|
|
161
|
+
puts "Accepting subscription request from #{presence.from}"
|
162
|
+
roster.accept_subscription(presence.from)
|
163
|
+
|
164
|
+
roster.add(presence.from.strip, presence.from.node, true)
|
165
|
+
}
|
166
|
+
|
167
|
+
@chatdialogs = {}
|
168
|
+
@chatdialogs_lock = Mutex.new
|
169
|
+
|
170
|
+
cl.add_message_callback { |msg|
|
171
|
+
puts "Message #{msg.type} from #{msg.from}: #{msg.body.inspect}"
|
172
|
+
|
173
|
+
if msg.type == :chat and msg.body
|
174
|
+
@chatdialogs_lock.synchronize {
|
175
|
+
unless @chatdialogs.has_key? msg.from
|
176
|
+
@chatdialogs[msg.from] = Interview.new(dbi, msg.from.strip.to_s, collections) { |str|
|
177
|
+
cl.send Jabber::Message.new(msg.from, str).set_type(:chat)
|
178
|
+
}
|
179
|
+
else
|
180
|
+
@chatdialogs[msg.from].on_message msg.body
|
181
|
+
end
|
182
|
+
|
183
|
+
@chatdialogs.delete_if { |jid,interview| interview.finished? }
|
184
|
+
}
|
185
|
+
end
|
186
|
+
}
|
187
|
+
|
188
|
+
cl.add_iq_callback { |iq|
|
189
|
+
answer = iq.answer
|
190
|
+
answer.type = :result
|
191
|
+
|
192
|
+
command = answer.first_element('command')
|
193
|
+
|
194
|
+
if iq.type == :get and iq.query.kind_of? Jabber::Discovery::IqQueryDiscoInfo
|
195
|
+
if iq.query.node == 'config'
|
196
|
+
answer.query.add Jabber::Discovery::Identity.new('automation', 'Configure subscriptions', 'command-node')
|
197
|
+
[ 'jabber:x:data',
|
198
|
+
'http://jabber.org/protocol/commands'].each { |feature|
|
199
|
+
answer.query.add Jabber::Discovery::Feature.new(feature)
|
200
|
+
}
|
201
|
+
else
|
202
|
+
answer.query.add Jabber::Discovery::Identity.new('headline', 'Harvester Jabber service', 'rss')
|
203
|
+
[ Jabber::Discovery::IqQueryDiscoInfo.new.namespace,
|
204
|
+
Jabber::Discovery::IqQueryDiscoItems.new.namespace,
|
205
|
+
'http://jabber.org/protocol/commands'].each { |feature|
|
206
|
+
answer.query.add Jabber::Discovery::Feature.new(feature)
|
207
|
+
}
|
208
|
+
end
|
209
|
+
elsif iq.type == :get and iq.query.kind_of? Jabber::Discovery::IqQueryDiscoItems
|
210
|
+
if iq.query.node == 'http://jabber.org/protocol/commands'
|
211
|
+
answer.query.add Jabber::Discovery::Item.new(cl.jid, 'Configure subscriptions', 'config')
|
212
|
+
else
|
213
|
+
answer.query.add Jabber::Discovery::Item.new(cl.jid, 'Ad-hoc commands', 'http://jabber.org/protocol/commands')
|
214
|
+
end
|
215
|
+
elsif iq.type == :set and command and command.namespace == 'http://jabber.org/protocol/commands' and command.attributes['node'] == 'config'
|
216
|
+
x = command.first_element('x')
|
217
|
+
x = Jabber::Dataforms::XData.new.import(x) if x
|
218
|
+
|
219
|
+
user = iq.from.strip.to_s
|
220
|
+
|
221
|
+
if x.nil? or x.type != :submit
|
222
|
+
puts "#{iq.from} requested data form"
|
223
|
+
command.attributes['status'] = 'executing'
|
224
|
+
command.attributes['sessionid'] = Jabber::IdGenerator.instance.generate_id
|
225
|
+
x = command.add(Jabber::Dataforms::XData.new(:form))
|
226
|
+
x.add(Jabber::Dataforms::XDataTitle.new).text = 'Configure subscriptions'
|
227
|
+
|
228
|
+
respect_status = x.add(Jabber::Dataforms::XDataField.new('respect-status', :boolean))
|
229
|
+
respect_status.label = 'Respect your online status'
|
230
|
+
message_type = x.add(Jabber::Dataforms::XDataField.new('message-type', :list_single))
|
231
|
+
message_type.label = 'Message type of notifications'
|
232
|
+
message_type.options = {'normal'=>'Normal message',
|
233
|
+
'chat'=>'Chat message',
|
234
|
+
'headline'=>'Headline message'}
|
235
|
+
settings = dbi.execute "SELECT respect_status, message_type FROM #{TABLE_SETTINGS} WHERE jid=?", user
|
236
|
+
while setting = settings.fetch
|
237
|
+
respect_status.values = [(setting.shift ? '1' : '0')]
|
238
|
+
message_type.values = [setting.shift]
|
239
|
+
end
|
240
|
+
|
241
|
+
collections.keys.sort.each { |collection|
|
242
|
+
field = x.add(Jabber::Dataforms::XDataField.new("collection-#{collection}", :boolean))
|
243
|
+
field.label = "Receive notifications for collection #{collection}"
|
244
|
+
field.add(REXML::Element.new('desc')).text = collections[collection].collect { |rss,title| title }.join(', ')
|
245
|
+
|
246
|
+
field.values = ['0']
|
247
|
+
subscription = dbi.execute "SELECT jid FROM #{TABLE_SUBSCRIPTIONS} WHERE jid=? AND collection=?", user, collection
|
248
|
+
while subscription.fetch
|
249
|
+
field.values = ['1']
|
250
|
+
end
|
251
|
+
}
|
252
|
+
else
|
253
|
+
if x and x.type == :submit
|
254
|
+
puts "#{iq.from} submitted data form"
|
255
|
+
|
256
|
+
if x.field('respect-status') and x.field('message-type')
|
257
|
+
respect_status = x.field('respect-status').values.include? '1'
|
258
|
+
message_type = x.field('message-type').values.to_s
|
259
|
+
|
260
|
+
dbi.do "DELETE FROM #{TABLE_SETTINGS} WHERE jid=?", user
|
261
|
+
dbi.do "INSERT INTO #{TABLE_SETTINGS} (jid, respect_status, message_type) VALUES (?, ?, ?)",
|
262
|
+
user, respect_status, message_type
|
263
|
+
end
|
264
|
+
|
265
|
+
x.each_element('field') { |f|
|
266
|
+
if f.var =~ /^collection-(.+)$/
|
267
|
+
collection = $1
|
268
|
+
dbi.execute "DELETE FROM #{TABLE_SUBSCRIPTIONS} WHERE jid=? AND collection=?", user, collection
|
269
|
+
if f.values.to_s == '1'
|
270
|
+
dbi.do "INSERT INTO #{TABLE_SUBSCRIPTIONS} (jid, collection) VALUES (?, ?)", user, collection
|
271
|
+
end
|
272
|
+
end
|
273
|
+
}
|
274
|
+
|
275
|
+
command.delete_element 'x'
|
276
|
+
command.attributes['status'] = 'completed'
|
277
|
+
note = command.add(REXML::Element.new('note'))
|
278
|
+
note.attributes['type'] = 'info'
|
279
|
+
note.text = 'Thank you for making use of the advanced NotAstroBot configuration interface. You are truly worth being notified about all that hot stuff!'
|
280
|
+
else
|
281
|
+
# Do nothing, but send a result
|
282
|
+
puts "#{iq.from} #{command.attributes['action']} data form"
|
283
|
+
|
284
|
+
command.delete_element 'x'
|
285
|
+
command.attributes['status'] = 'canceled'
|
286
|
+
end
|
287
|
+
end
|
288
|
+
elsif iq.type == :get or iq.type == :get
|
289
|
+
answer.type = :error
|
290
|
+
answer.add Jabber::ErrorResponse.new('feature-not-implemented', 'The requested feature hasn\'t been implemented.')
|
291
|
+
else
|
292
|
+
answer = ' '
|
293
|
+
end
|
294
|
+
|
295
|
+
cl.send answer
|
296
|
+
}
|
297
|
+
|
298
|
+
cl.send Jabber::Presence.new(:chat, 'The modern Harvester Jabber Service (Public Beta)')
|
299
|
+
|
300
|
+
messages_sent = 0
|
301
|
+
startup = Time.new
|
302
|
+
links = []
|
303
|
+
dbi.execute("SELECT link FROM last48hrs").each { |link,|
|
304
|
+
links << link
|
305
|
+
}
|
306
|
+
|
307
|
+
chart_last_update = Time.at(0)
|
308
|
+
chart_filename = "#{config['settings']['output']}/chart.jpg"
|
309
|
+
avatar_hash = ""
|
310
|
+
|
311
|
+
loop {
|
312
|
+
resend_presence = false
|
313
|
+
|
314
|
+
###
|
315
|
+
# Update collections
|
316
|
+
###
|
317
|
+
new_collections = Hash.new([])
|
318
|
+
|
319
|
+
sources = dbi.execute "SELECT collection,rss,title FROM sources ORDER BY collection,title"
|
320
|
+
while row = sources.fetch
|
321
|
+
collection, rss, title = row
|
322
|
+
new_collections[collection] += [[rss, title]]
|
323
|
+
end
|
324
|
+
|
325
|
+
collections = new_collections
|
326
|
+
|
327
|
+
###
|
328
|
+
# Find new items
|
329
|
+
##
|
330
|
+
# This fetches all items from the last 48 hours,
|
331
|
+
# just to make sure to not miss anything due to
|
332
|
+
# timezone overlaps and so on.
|
333
|
+
###
|
334
|
+
new_links = []
|
335
|
+
notifications = Hash.new([])
|
336
|
+
items = dbi.execute "SELECT rss, blogtitle, title, link, collection FROM last48hrs"
|
337
|
+
while row = items.fetch
|
338
|
+
rss, blogtitle, title, link, collection = row
|
339
|
+
|
340
|
+
unless links.include? link
|
341
|
+
puts "New: #{link} (#{blogtitle}: #{title})"
|
342
|
+
notifications[collection] += [[blogtitle, title, link]]
|
343
|
+
|
344
|
+
resend_presence = true
|
345
|
+
end
|
346
|
+
|
347
|
+
new_links << link
|
348
|
+
end
|
349
|
+
|
350
|
+
notifications.keys.each { |collection|
|
351
|
+
text = "Updates for #{collection}:"
|
352
|
+
subject = []
|
353
|
+
|
354
|
+
html = REXML::Element.new 'html'
|
355
|
+
html.add_namespace 'http://jabber.org/protocol/xhtml-im'
|
356
|
+
body = html.add REXML::Element.new('body')
|
357
|
+
body.add_namespace 'http://www.w3.org/1999/xhtml'
|
358
|
+
body.add(REXML::Element.new('h4')).text = "Updates for #{collection}"
|
359
|
+
ul = body.add(REXML::Element.new('ul'))
|
360
|
+
|
361
|
+
notifications[collection].each { |blogtitle, title, link|
|
362
|
+
subject << blogtitle
|
363
|
+
text += "\n#{blogtitle}: #{title}\n#{link}"
|
364
|
+
|
365
|
+
li = ul.add(REXML::Element.new('li'))
|
366
|
+
li.add REXML::Text.new("#{blogtitle}: ")
|
367
|
+
a = li.add(REXML::Element.new('a'))
|
368
|
+
a.attributes['href'] = link
|
369
|
+
a.text = title
|
370
|
+
}
|
371
|
+
|
372
|
+
puts "#{Time.new} - #{text.inspect}"
|
373
|
+
|
374
|
+
##
|
375
|
+
# Prepare subject
|
376
|
+
subject.uniq!
|
377
|
+
subject.sort! { |a,b| a.downcase <=> b.downcase }
|
378
|
+
|
379
|
+
##
|
380
|
+
# Send for all who have subscribed
|
381
|
+
subscriptions = dbi.execute "SELECT jid FROM #{TABLE_SUBSCRIPTIONS} WHERE collection=?", collection
|
382
|
+
while row = subscriptions.fetch
|
383
|
+
jid, = row
|
384
|
+
|
385
|
+
respect_status = false
|
386
|
+
message_type = :headline
|
387
|
+
settings = dbi.execute "SELECT respect_status, message_type FROM #{TABLE_SETTINGS} WHERE jid=?", jid
|
388
|
+
while setting = settings.fetch
|
389
|
+
respect_status = setting.shift
|
390
|
+
message_type = setting.shift.intern
|
391
|
+
end
|
392
|
+
|
393
|
+
if (respect_status and (roster[jid] ? roster[jid].online? : false)) or not respect_status
|
394
|
+
msg = Jabber::Message.new
|
395
|
+
msg.to, = jid
|
396
|
+
msg.type = message_type
|
397
|
+
msg.subject = subject.join', '
|
398
|
+
msg.body = text
|
399
|
+
msg.add html
|
400
|
+
cl.send msg
|
401
|
+
end
|
402
|
+
|
403
|
+
messages_sent += 1
|
404
|
+
end
|
405
|
+
}
|
406
|
+
|
407
|
+
links = new_links
|
408
|
+
|
409
|
+
##
|
410
|
+
# Avatar
|
411
|
+
##
|
412
|
+
if File::ctime(chart_filename) > chart_last_update
|
413
|
+
chart_last_update = File::ctime(chart_filename)
|
414
|
+
|
415
|
+
photo = IO::readlines(chart_filename).to_s
|
416
|
+
avatar_hash = Digest::SHA1.hexdigest(photo)
|
417
|
+
vcard = Jabber::Vcard::IqVcard.new('NICKNAME' => 'NotAstrobot',
|
418
|
+
'FN' => 'Harvester Jabber notification',
|
419
|
+
'URL' => 'http://localhost/',
|
420
|
+
'PHOTO/TYPE' => 'image/jpeg',
|
421
|
+
'PHOTO/BINVAL' => Base64::encode64(photo))
|
422
|
+
Jabber::Vcard::Helper::set(cl, vcard)
|
423
|
+
resend_presence = true
|
424
|
+
end
|
425
|
+
|
426
|
+
if resend_presence
|
427
|
+
pres = Jabber::Presence.new(:chat,
|
428
|
+
"Sent #{messages_sent} messages in #{duration_to_s(Time.new - startup)}. Chewed #{links.size} feed items in the last 48 hours.")
|
429
|
+
x = pres.add(REXML::Element.new('x'))
|
430
|
+
x.add_namespace 'vcard-temp:x:update'
|
431
|
+
x.add(REXML::Element.new('photo')).text = avatar_hash
|
432
|
+
cl.send pres
|
433
|
+
end
|
434
|
+
|
435
|
+
###
|
436
|
+
# Loop
|
437
|
+
###
|
438
|
+
print '.'; $stdout.flush
|
439
|
+
sleep config['jabber']['interval'].to_i
|
440
|
+
}
|
441
|
+
|
442
|
+
end
|
443
|
+
end
|