astrotrain 0.5.4 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +8 -0
- data/LICENSE +18 -17
- data/Rakefile +118 -103
- data/astrotrain.gemspec +87 -136
- data/lib/astrotrain.rb +47 -51
- data/lib/astrotrain/attachment.rb +55 -0
- data/lib/astrotrain/message.rb +221 -235
- data/lib/astrotrain/transports/http_post.rb +67 -0
- data/lib/astrotrain/transports/resque.rb +63 -0
- data/test/fixtures/bad_email_format.txt +15 -0
- data/test/fixtures/basic.txt +4 -1
- data/test/fixtures/iso-8859-1.txt +1 -0
- data/test/message_test.rb +146 -457
- data/test/test_helper.rb +20 -42
- data/test/transport_test.rb +98 -100
- metadata +100 -243
- data/.gitignore +0 -26
- data/README +0 -47
- data/VERSION +0 -1
- data/config/sample.rb +0 -12
- data/lib/astrotrain/api.rb +0 -52
- data/lib/astrotrain/logged_mail.rb +0 -48
- data/lib/astrotrain/mapping.rb +0 -162
- data/lib/astrotrain/mapping/http_post.rb +0 -18
- data/lib/astrotrain/mapping/jabber.rb +0 -28
- data/lib/astrotrain/mapping/transport.rb +0 -55
- data/lib/astrotrain/tmail.rb +0 -58
- data/lib/astrotrain/worker.rb +0 -65
- data/lib/vendor/rest-client/README.rdoc +0 -104
- data/lib/vendor/rest-client/Rakefile +0 -84
- data/lib/vendor/rest-client/bin/restclient +0 -65
- data/lib/vendor/rest-client/foo.diff +0 -66
- data/lib/vendor/rest-client/lib/rest_client.rb +0 -188
- data/lib/vendor/rest-client/lib/rest_client/net_http_ext.rb +0 -23
- data/lib/vendor/rest-client/lib/rest_client/payload.rb +0 -185
- data/lib/vendor/rest-client/lib/rest_client/request_errors.rb +0 -75
- data/lib/vendor/rest-client/lib/rest_client/resource.rb +0 -103
- data/lib/vendor/rest-client/rest-client.gemspec +0 -18
- data/lib/vendor/rest-client/spec/base.rb +0 -5
- data/lib/vendor/rest-client/spec/master_shake.jpg +0 -0
- data/lib/vendor/rest-client/spec/payload_spec.rb +0 -71
- data/lib/vendor/rest-client/spec/request_errors_spec.rb +0 -44
- data/lib/vendor/rest-client/spec/resource_spec.rb +0 -52
- data/lib/vendor/rest-client/spec/rest_client_spec.rb +0 -219
- data/test/api_test.rb +0 -32
- data/test/logged_mail_test.rb +0 -67
- data/test/mapping_test.rb +0 -129
data/.gitignore
DELETED
@@ -1,26 +0,0 @@
|
|
1
|
-
.DS_Store
|
2
|
-
.rake_tasks
|
3
|
-
log/*
|
4
|
-
tmp/*
|
5
|
-
TAGS
|
6
|
-
*~
|
7
|
-
.#*
|
8
|
-
pkg
|
9
|
-
schema/schema.rb
|
10
|
-
schema/*_structure.sql
|
11
|
-
schema/*.sqlite3
|
12
|
-
schema/*.sqlite
|
13
|
-
schema/*.db
|
14
|
-
*.sqlite
|
15
|
-
*.sqlite3
|
16
|
-
*.db
|
17
|
-
src/*
|
18
|
-
.hgignore
|
19
|
-
.hg/*
|
20
|
-
.svn/*
|
21
|
-
doc
|
22
|
-
config/database.yml
|
23
|
-
merb/custom.rb
|
24
|
-
test/fixtures/queue
|
25
|
-
messages
|
26
|
-
test/messages
|
data/README
DELETED
@@ -1,47 +0,0 @@
|
|
1
|
-
astrotrain
|
2
|
-
==========
|
3
|
-
|
4
|
-
NOTE: Astrotrain is a full gem now. If you're looking for the old Astrotrain on merb:
|
5
|
-
http://github.com/entp/astrotrain/tree/merb
|
6
|
-
git://github.com/entp/astrotrain.git (merb branch)
|
7
|
-
|
8
|
-
Scans incoming emails for mapped recipients and sends an HTTP POST somewhere.
|
9
|
-
|
10
|
-
# setup a config file.
|
11
|
-
# Point the queue_path at a directory that your mail server dumps each raw incoming mail.
|
12
|
-
require 'astrotrain'
|
13
|
-
|
14
|
-
Astrotrain.load path do
|
15
|
-
DataMapper.setup(:default, {
|
16
|
-
:adapter => "mysql",
|
17
|
-
:database => "astrotrain",
|
18
|
-
:username => "root",
|
19
|
-
:host => "localhost"
|
20
|
-
})
|
21
|
-
end
|
22
|
-
Astrotrain::Message.queue_path = "/path/to/maildir"
|
23
|
-
Astrotrain::Mapping::Transport.processing = true
|
24
|
-
|
25
|
-
# start up IRB
|
26
|
-
irb -I /var/astrotrain/lib -r config.rb
|
27
|
-
|
28
|
-
# manage mappings
|
29
|
-
LIB=/var/astrotrain/lib CONFIG=config.rb rake at:mappings
|
30
|
-
LIB=/var/astrotrain/lib CONFIG=config.rb rake at:map EMAIL=support@foo.com DEST=http://foo.com/email
|
31
|
-
LIB=/var/astrotrain/lib CONFIG=config.rb rake at:unmap MAP=123
|
32
|
-
|
33
|
-
# start the server that runs over the queue directory
|
34
|
-
LIB=/var/astrotrain/lib CONFIG=config.rb rake at:process
|
35
|
-
|
36
|
-
# start the sinatra API
|
37
|
-
ruby -I /var/astrotrain/lib /var/astrotrain/lib/astrotrain/api.rb config.rb
|
38
|
-
|
39
|
-
A single Astrotrain process currently handles email for two production applications, processing thousands daily.
|
40
|
-
It's far from perfect, but definitely usable.
|
41
|
-
|
42
|
-
TODO
|
43
|
-
====
|
44
|
-
|
45
|
-
Docs
|
46
|
-
bounced emails (i've been rolling with http://github.com/whatcould/bounce-email for now)
|
47
|
-
Mail gem, TMail is old school
|
data/VERSION
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
0.5.4
|
data/config/sample.rb
DELETED
@@ -1,12 +0,0 @@
|
|
1
|
-
path = File.join(File.dirname(__FILE__), '..')
|
2
|
-
$LOAD_PATH.unshift File.join(path, 'lib')
|
3
|
-
require 'astrotrain'
|
4
|
-
|
5
|
-
Astrotrain.load path do
|
6
|
-
DataMapper.setup(:default, {
|
7
|
-
:adapter => "mysql",
|
8
|
-
:database => "astrotrain",
|
9
|
-
:username => "root",
|
10
|
-
:host => "localhost"
|
11
|
-
})
|
12
|
-
end
|
data/lib/astrotrain/api.rb
DELETED
@@ -1,52 +0,0 @@
|
|
1
|
-
require 'sinatra'
|
2
|
-
|
3
|
-
before do
|
4
|
-
response['Content-Type'] = 'text/plain'
|
5
|
-
end
|
6
|
-
|
7
|
-
get '/queue_size' do
|
8
|
-
if File.exist?(Astrotrain::Message.queue_path)
|
9
|
-
(Dir.entries(Astrotrain::Message.queue_path).size - 2).to_s
|
10
|
-
else
|
11
|
-
'0'
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
get '/queue/*' do
|
16
|
-
path = params[:splat].first
|
17
|
-
path.gsub! /^\/+/, ''
|
18
|
-
file = File.join(Astrotrain::Message.queue_path, path)
|
19
|
-
if File.exist?(file)
|
20
|
-
IO.read(file)
|
21
|
-
else
|
22
|
-
halt 404, "#{path.inspect} was not found."
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
get '/queue' do
|
27
|
-
data = []
|
28
|
-
Dir.entries(Astrotrain::Message.queue_path).each do |e|
|
29
|
-
data << e unless e =~ /^\.{1,2}$/
|
30
|
-
end
|
31
|
-
data * "\n"
|
32
|
-
end
|
33
|
-
|
34
|
-
unless $testing
|
35
|
-
configure do
|
36
|
-
# the idea being, you pass the name of the config file
|
37
|
-
#
|
38
|
-
# # loads File.join(Dir.pwd, 'config.rb')
|
39
|
-
# CONFIG=config ruby lib/astrotrain/api.rb
|
40
|
-
#
|
41
|
-
# # loads File.join(Dir.pwd, 'config.rb')
|
42
|
-
# ruby lib/astrotrain/api.rb config
|
43
|
-
#
|
44
|
-
# That file contains the Greymalkin.load block that initializes Sequel
|
45
|
-
#
|
46
|
-
path = ENV['CONFIG'] || ARGV.shift
|
47
|
-
if path[0..0] != '/'
|
48
|
-
path = File.join(Dir.pwd, path)
|
49
|
-
end
|
50
|
-
require File.expand_path(path)
|
51
|
-
end
|
52
|
-
end
|
@@ -1,48 +0,0 @@
|
|
1
|
-
module Astrotrain
|
2
|
-
# Logs details of each incoming message.
|
3
|
-
class LoggedMail
|
4
|
-
include DataMapper::Resource
|
5
|
-
|
6
|
-
class << self
|
7
|
-
attr_accessor :log_path, :log_processed
|
8
|
-
end
|
9
|
-
|
10
|
-
# Enabling this will save records for every processed email, not just the errored emails.
|
11
|
-
self.log_processed = false
|
12
|
-
self.log_path = File.join(Astrotrain.root, 'messages')
|
13
|
-
|
14
|
-
property :id, Serial
|
15
|
-
property :mapping_id, Integer, :index => true
|
16
|
-
property :message_id, String, :index => true, :size => 255, :length => 1..255
|
17
|
-
property :sender, String, :index => true, :size => 255, :length => 1..255
|
18
|
-
property :recipient, String, :index => true, :size => 255, :length => 1..255
|
19
|
-
property :subject, String, :index => true, :size => 255, :length => 1..255
|
20
|
-
property :mail_file, String, :size => 255, :length => 1..255
|
21
|
-
property :created_at, DateTime
|
22
|
-
property :delivered_at, DateTime
|
23
|
-
property :error_message, Text
|
24
|
-
|
25
|
-
belongs_to :mapping
|
26
|
-
|
27
|
-
def self.from(message, file = nil)
|
28
|
-
logged = new
|
29
|
-
begin
|
30
|
-
logged.message_id = message.message_id
|
31
|
-
logged.sender = Message.parse_email_addresses(message.sender).first
|
32
|
-
logged.subject = message.subject
|
33
|
-
logged.mail_file = file if file
|
34
|
-
end
|
35
|
-
if !block_given? || yield(logged)
|
36
|
-
begin
|
37
|
-
logged.save
|
38
|
-
if logged.delivered_at && File.exist?(logged.mail_file.to_s)
|
39
|
-
FileUtils.rm_rf logged.mail_file
|
40
|
-
end
|
41
|
-
rescue
|
42
|
-
puts $!.inspect
|
43
|
-
end
|
44
|
-
end
|
45
|
-
logged
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
data/lib/astrotrain/mapping.rb
DELETED
@@ -1,162 +0,0 @@
|
|
1
|
-
module Astrotrain
|
2
|
-
class Mapping
|
3
|
-
include DataMapper::Resource
|
4
|
-
|
5
|
-
class << self
|
6
|
-
attr_accessor :default_domain
|
7
|
-
attr_accessor :transports
|
8
|
-
end
|
9
|
-
|
10
|
-
self.transports = {"HTTP Post" => 'http_post', "Jabber" => 'jabber'}
|
11
|
-
self.default_domain = 'astrotrain.com'
|
12
|
-
|
13
|
-
property :id, Serial
|
14
|
-
property :email_user, String, :size => 255, :length => 1..255, :index => :email, :format => /^[\w\.\_\%\+\-]*\*?$/
|
15
|
-
property :email_domain, String, :size => 255, :lenght => 1..255, :index => :email, :format => /^[\w\-\_\.]+$/, :default => lambda { default_domain }
|
16
|
-
property :destination, String, :size => 255, :length => 1..255
|
17
|
-
property :transport, String, :size => 255, :set => transports.values, :default => 'http_post'
|
18
|
-
property :separator, String, :size => 255
|
19
|
-
|
20
|
-
validates_is_unique :email_user, :scope => :email_domain
|
21
|
-
validates_format :destination, :as => /^(https?:)\/\/[^\/]+\/?/i, :if => :destination_uses_url?
|
22
|
-
validates_format :destination, :as => :email_address, :if => :destination_uses_email?
|
23
|
-
|
24
|
-
has n, :logged_mails, :order => [:created_at.desc]
|
25
|
-
|
26
|
-
# returns a mapping for the given array of email addresses
|
27
|
-
def self.match(email_addresses)
|
28
|
-
email_addresses.each do |email_address|
|
29
|
-
email_address.strip!
|
30
|
-
email_address.downcase!
|
31
|
-
name, domain = email_address.split("@")
|
32
|
-
if mapping = match_by_address(name, domain) || match_by_wildcard(name, domain)
|
33
|
-
return [mapping, email_address]
|
34
|
-
end
|
35
|
-
end
|
36
|
-
nil
|
37
|
-
end
|
38
|
-
|
39
|
-
# Processes a given message. It finds a mapping, creates a LoggedMail record,
|
40
|
-
# and attempts to process the message.
|
41
|
-
def self.process(message, file = nil)
|
42
|
-
LoggedMail.from(message, file) do |logged|
|
43
|
-
save_logged = begin
|
44
|
-
mapping, recipient = match(message.recipients)
|
45
|
-
if mapping
|
46
|
-
logged.recipient = recipient
|
47
|
-
logged.mapping = mapping
|
48
|
-
begin
|
49
|
-
mapping.process(message, recipient)
|
50
|
-
rescue Astrotrain::ProcessingCancelled
|
51
|
-
logged.error_message = "Cancelled."
|
52
|
-
end
|
53
|
-
logged.delivered_at = Time.now.utc
|
54
|
-
end
|
55
|
-
LoggedMail.log_processed # save successfully processed messages?
|
56
|
-
rescue
|
57
|
-
logged.error_message = "#{$!.message}\n#{$!.backtrace.join("\n")}"
|
58
|
-
end
|
59
|
-
Astrotrain.callback(:post_processing, message, mapping, logged)
|
60
|
-
save_logged
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
|
-
# Processes a given message and recipient against the mapping's transport.
|
65
|
-
def process(message, recipient)
|
66
|
-
Astrotrain.callback(:pre_processing, message, self)
|
67
|
-
Transport.process(message, self, recipient)
|
68
|
-
|
69
|
-
end
|
70
|
-
|
71
|
-
# returns true if the email matches this mapping. Wildcards in the name are allowed.
|
72
|
-
# A mapping with foo*@bar.com will match foo@bar.com and food@bar.com, but not foo@baz.com.
|
73
|
-
def match?(name, domain)
|
74
|
-
email_domain == domain && name =~ email_user_regex
|
75
|
-
end
|
76
|
-
|
77
|
-
def destination_uses_url?
|
78
|
-
transport == 'http_post'
|
79
|
-
end
|
80
|
-
|
81
|
-
def destination_uses_email?
|
82
|
-
transport == 'jabber'
|
83
|
-
end
|
84
|
-
|
85
|
-
def full_email
|
86
|
-
"#{email_user}@#{email_domain}"
|
87
|
-
end
|
88
|
-
|
89
|
-
# Looks for the mapping's separator in the message body and pulls only the content
|
90
|
-
# above it. Assuming a separator of '===='...
|
91
|
-
#
|
92
|
-
# This will be kept
|
93
|
-
#
|
94
|
-
# On Thu, Sep 3, 2009 at 12:34 AM... (this will be removed)
|
95
|
-
# ====
|
96
|
-
#
|
97
|
-
# > Everything here will be removed.
|
98
|
-
#
|
99
|
-
def find_reply_from(body)
|
100
|
-
return if separator.blank?
|
101
|
-
return '' if body.blank?
|
102
|
-
lines = body.split("\n")
|
103
|
-
delim_line = found_empty = nil
|
104
|
-
|
105
|
-
(lines.size - 1).downto(0) do |i|
|
106
|
-
line = lines[i]
|
107
|
-
if !delim_line && line.include?(separator)
|
108
|
-
delim_line = i
|
109
|
-
elsif delim_line && !found_empty
|
110
|
-
delim_line = i
|
111
|
-
found_empty = line.strip.blank?
|
112
|
-
elsif delim_line && found_empty
|
113
|
-
if date_reply_line?(line) || line.strip.blank?
|
114
|
-
delim_line = i
|
115
|
-
else
|
116
|
-
break
|
117
|
-
end
|
118
|
-
end
|
119
|
-
end
|
120
|
-
|
121
|
-
if delim_line
|
122
|
-
body = if delim_line.zero?
|
123
|
-
[]
|
124
|
-
elsif lines.size >= delim_line
|
125
|
-
lines[0..delim_line-1]
|
126
|
-
else
|
127
|
-
lines
|
128
|
-
end.join("\n")
|
129
|
-
elsif body.frozen?
|
130
|
-
body = body.dup
|
131
|
-
end
|
132
|
-
body.strip!
|
133
|
-
body
|
134
|
-
end
|
135
|
-
|
136
|
-
protected
|
137
|
-
def self.match_by_address(name, domain)
|
138
|
-
first(:email_user => name, :email_domain => domain)
|
139
|
-
end
|
140
|
-
|
141
|
-
def self.match_by_wildcard(name, domain)
|
142
|
-
wildcards = all(:email_domain => domain, :email_user.like => "%*")
|
143
|
-
wildcards.sort! { |x, y| y.email_user.size <=> x.email_user.size }
|
144
|
-
wildcards.detect { |w| w.match?(name, domain) }
|
145
|
-
end
|
146
|
-
|
147
|
-
DATE_LANGUATE_REGEXES = [/^on\b.*wrote\b?:$/i, /^am\b.*schrieb [\w\d\s]+:$/i, /^le\b.*a écrit\b?:$/i]
|
148
|
-
def date_reply_line?(line)
|
149
|
-
DATE_LANGUATE_REGEXES.any? { |re| line =~ re }
|
150
|
-
end
|
151
|
-
|
152
|
-
def email_user_regex
|
153
|
-
@email_user_regex ||= begin
|
154
|
-
if email_user['*']
|
155
|
-
/^#{email_user.sub /\*/, '(.*)'}$/
|
156
|
-
else
|
157
|
-
/^#{email_user}$/
|
158
|
-
end
|
159
|
-
end
|
160
|
-
end
|
161
|
-
end
|
162
|
-
end
|
@@ -1,18 +0,0 @@
|
|
1
|
-
module Astrotrain
|
2
|
-
class Mapping
|
3
|
-
class HttpPost < Transport
|
4
|
-
def process
|
5
|
-
return unless Transport.processing
|
6
|
-
RestClient.post @mapping.destination, fields.merge(:emails => fields[:emails].join(","))
|
7
|
-
end
|
8
|
-
|
9
|
-
def fields
|
10
|
-
super
|
11
|
-
@message.attachments.each_with_index do |att, index|
|
12
|
-
@fields[:"attachments_#{index}"] = att
|
13
|
-
end
|
14
|
-
@fields
|
15
|
-
end
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|
@@ -1,28 +0,0 @@
|
|
1
|
-
begin
|
2
|
-
require 'xmpp4r-simple'
|
3
|
-
module Astrotrain
|
4
|
-
class Mapping
|
5
|
-
# This is experimental. No attachments are supported.
|
6
|
-
class Jabber < Transport
|
7
|
-
class << self
|
8
|
-
attr_accessor :login, :password
|
9
|
-
end
|
10
|
-
|
11
|
-
def process
|
12
|
-
return unless Transport.processing
|
13
|
-
connection.deliver(@mapping.destination, content)
|
14
|
-
end
|
15
|
-
|
16
|
-
def connection
|
17
|
-
@connection ||= ::Jabber::Simple.new(self.class.login, self.class.password)
|
18
|
-
end
|
19
|
-
|
20
|
-
def content
|
21
|
-
@content ||= "From: %s\nTo: %s\nSubject: %s\nEmails: %s\n%s" % [fields[:from], fields[:to], fields[:subject], fields[:emails] * ", ", fields[:body]]
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
26
|
-
rescue LoadError
|
27
|
-
puts "Install xmpp4r-simple for Jabber support."
|
28
|
-
end
|
@@ -1,55 +0,0 @@
|
|
1
|
-
module Astrotrain
|
2
|
-
class Mapping
|
3
|
-
class Transport
|
4
|
-
class << self
|
5
|
-
attr_accessor :processing
|
6
|
-
end
|
7
|
-
|
8
|
-
# Enable this turn on processing.
|
9
|
-
self.processing = false
|
10
|
-
|
11
|
-
attr_reader :message, :mapping
|
12
|
-
|
13
|
-
# process a given message against the mapping. The mapping transport is checked,
|
14
|
-
# and the appropirate transport class handles the request.
|
15
|
-
def self.process(message, mapping, recipient)
|
16
|
-
case mapping.transport
|
17
|
-
when 'http_post' then HttpPost.process(message, mapping, recipient)
|
18
|
-
when 'jabber' then Jabber.process(message, mapping, recipient)
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
def initialize(message, mapping, recipient)
|
23
|
-
message.body = mapping.find_reply_from(message.body)
|
24
|
-
@message = message
|
25
|
-
@mapping = mapping
|
26
|
-
@recipient = recipient
|
27
|
-
end
|
28
|
-
|
29
|
-
def process
|
30
|
-
raise UnimplementedError
|
31
|
-
end
|
32
|
-
|
33
|
-
def fields
|
34
|
-
@fields ||= begin
|
35
|
-
all_emails = @message.recipients - [@recipient]
|
36
|
-
f = {:subject => @message.subject, :to => @recipient, :from => @message.sender, :body => @message.body, :emails => all_emails, :html => @message.html}
|
37
|
-
@message.headers.each do |key, value|
|
38
|
-
f["headers[#{key}]"] = value
|
39
|
-
end
|
40
|
-
f
|
41
|
-
end
|
42
|
-
end
|
43
|
-
|
44
|
-
# defines custom #process class methods that instantiate the class and calls a #process instance method
|
45
|
-
def self.inherited(child)
|
46
|
-
super
|
47
|
-
class << child
|
48
|
-
def process(message, mapping, recipient)
|
49
|
-
new(message, mapping, recipient).process
|
50
|
-
end
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|