astrotrain 0.5.4 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. data/Gemfile +8 -0
  2. data/LICENSE +18 -17
  3. data/Rakefile +118 -103
  4. data/astrotrain.gemspec +87 -136
  5. data/lib/astrotrain.rb +47 -51
  6. data/lib/astrotrain/attachment.rb +55 -0
  7. data/lib/astrotrain/message.rb +221 -235
  8. data/lib/astrotrain/transports/http_post.rb +67 -0
  9. data/lib/astrotrain/transports/resque.rb +63 -0
  10. data/test/fixtures/bad_email_format.txt +15 -0
  11. data/test/fixtures/basic.txt +4 -1
  12. data/test/fixtures/iso-8859-1.txt +1 -0
  13. data/test/message_test.rb +146 -457
  14. data/test/test_helper.rb +20 -42
  15. data/test/transport_test.rb +98 -100
  16. metadata +100 -243
  17. data/.gitignore +0 -26
  18. data/README +0 -47
  19. data/VERSION +0 -1
  20. data/config/sample.rb +0 -12
  21. data/lib/astrotrain/api.rb +0 -52
  22. data/lib/astrotrain/logged_mail.rb +0 -48
  23. data/lib/astrotrain/mapping.rb +0 -162
  24. data/lib/astrotrain/mapping/http_post.rb +0 -18
  25. data/lib/astrotrain/mapping/jabber.rb +0 -28
  26. data/lib/astrotrain/mapping/transport.rb +0 -55
  27. data/lib/astrotrain/tmail.rb +0 -58
  28. data/lib/astrotrain/worker.rb +0 -65
  29. data/lib/vendor/rest-client/README.rdoc +0 -104
  30. data/lib/vendor/rest-client/Rakefile +0 -84
  31. data/lib/vendor/rest-client/bin/restclient +0 -65
  32. data/lib/vendor/rest-client/foo.diff +0 -66
  33. data/lib/vendor/rest-client/lib/rest_client.rb +0 -188
  34. data/lib/vendor/rest-client/lib/rest_client/net_http_ext.rb +0 -23
  35. data/lib/vendor/rest-client/lib/rest_client/payload.rb +0 -185
  36. data/lib/vendor/rest-client/lib/rest_client/request_errors.rb +0 -75
  37. data/lib/vendor/rest-client/lib/rest_client/resource.rb +0 -103
  38. data/lib/vendor/rest-client/rest-client.gemspec +0 -18
  39. data/lib/vendor/rest-client/spec/base.rb +0 -5
  40. data/lib/vendor/rest-client/spec/master_shake.jpg +0 -0
  41. data/lib/vendor/rest-client/spec/payload_spec.rb +0 -71
  42. data/lib/vendor/rest-client/spec/request_errors_spec.rb +0 -44
  43. data/lib/vendor/rest-client/spec/resource_spec.rb +0 -52
  44. data/lib/vendor/rest-client/spec/rest_client_spec.rb +0 -219
  45. data/test/api_test.rb +0 -32
  46. data/test/logged_mail_test.rb +0 -67
  47. data/test/mapping_test.rb +0 -129
data/.gitignore DELETED
@@ -1,26 +0,0 @@
1
- .DS_Store
2
- .rake_tasks
3
- log/*
4
- tmp/*
5
- TAGS
6
- *~
7
- .#*
8
- pkg
9
- schema/schema.rb
10
- schema/*_structure.sql
11
- schema/*.sqlite3
12
- schema/*.sqlite
13
- schema/*.db
14
- *.sqlite
15
- *.sqlite3
16
- *.db
17
- src/*
18
- .hgignore
19
- .hg/*
20
- .svn/*
21
- doc
22
- config/database.yml
23
- merb/custom.rb
24
- test/fixtures/queue
25
- messages
26
- test/messages
data/README DELETED
@@ -1,47 +0,0 @@
1
- astrotrain
2
- ==========
3
-
4
- NOTE: Astrotrain is a full gem now. If you're looking for the old Astrotrain on merb:
5
- http://github.com/entp/astrotrain/tree/merb
6
- git://github.com/entp/astrotrain.git (merb branch)
7
-
8
- Scans incoming emails for mapped recipients and sends an HTTP POST somewhere.
9
-
10
- # setup a config file.
11
- # Point the queue_path at a directory that your mail server dumps each raw incoming mail.
12
- require 'astrotrain'
13
-
14
- Astrotrain.load path do
15
- DataMapper.setup(:default, {
16
- :adapter => "mysql",
17
- :database => "astrotrain",
18
- :username => "root",
19
- :host => "localhost"
20
- })
21
- end
22
- Astrotrain::Message.queue_path = "/path/to/maildir"
23
- Astrotrain::Mapping::Transport.processing = true
24
-
25
- # start up IRB
26
- irb -I /var/astrotrain/lib -r config.rb
27
-
28
- # manage mappings
29
- LIB=/var/astrotrain/lib CONFIG=config.rb rake at:mappings
30
- LIB=/var/astrotrain/lib CONFIG=config.rb rake at:map EMAIL=support@foo.com DEST=http://foo.com/email
31
- LIB=/var/astrotrain/lib CONFIG=config.rb rake at:unmap MAP=123
32
-
33
- # start the server that runs over the queue directory
34
- LIB=/var/astrotrain/lib CONFIG=config.rb rake at:process
35
-
36
- # start the sinatra API
37
- ruby -I /var/astrotrain/lib /var/astrotrain/lib/astrotrain/api.rb config.rb
38
-
39
- A single Astrotrain process currently handles email for two production applications, processing thousands daily.
40
- It's far from perfect, but definitely usable.
41
-
42
- TODO
43
- ====
44
-
45
- Docs
46
- bounced emails (i've been rolling with http://github.com/whatcould/bounce-email for now)
47
- Mail gem, TMail is old school
data/VERSION DELETED
@@ -1 +0,0 @@
1
- 0.5.4
data/config/sample.rb DELETED
@@ -1,12 +0,0 @@
1
- path = File.join(File.dirname(__FILE__), '..')
2
- $LOAD_PATH.unshift File.join(path, 'lib')
3
- require 'astrotrain'
4
-
5
- Astrotrain.load path do
6
- DataMapper.setup(:default, {
7
- :adapter => "mysql",
8
- :database => "astrotrain",
9
- :username => "root",
10
- :host => "localhost"
11
- })
12
- end
@@ -1,52 +0,0 @@
1
- require 'sinatra'
2
-
3
- before do
4
- response['Content-Type'] = 'text/plain'
5
- end
6
-
7
- get '/queue_size' do
8
- if File.exist?(Astrotrain::Message.queue_path)
9
- (Dir.entries(Astrotrain::Message.queue_path).size - 2).to_s
10
- else
11
- '0'
12
- end
13
- end
14
-
15
- get '/queue/*' do
16
- path = params[:splat].first
17
- path.gsub! /^\/+/, ''
18
- file = File.join(Astrotrain::Message.queue_path, path)
19
- if File.exist?(file)
20
- IO.read(file)
21
- else
22
- halt 404, "#{path.inspect} was not found."
23
- end
24
- end
25
-
26
- get '/queue' do
27
- data = []
28
- Dir.entries(Astrotrain::Message.queue_path).each do |e|
29
- data << e unless e =~ /^\.{1,2}$/
30
- end
31
- data * "\n"
32
- end
33
-
34
- unless $testing
35
- configure do
36
- # the idea being, you pass the name of the config file
37
- #
38
- # # loads File.join(Dir.pwd, 'config.rb')
39
- # CONFIG=config ruby lib/astrotrain/api.rb
40
- #
41
- # # loads File.join(Dir.pwd, 'config.rb')
42
- # ruby lib/astrotrain/api.rb config
43
- #
44
- # That file contains the Greymalkin.load block that initializes Sequel
45
- #
46
- path = ENV['CONFIG'] || ARGV.shift
47
- if path[0..0] != '/'
48
- path = File.join(Dir.pwd, path)
49
- end
50
- require File.expand_path(path)
51
- end
52
- end
@@ -1,48 +0,0 @@
1
- module Astrotrain
2
- # Logs details of each incoming message.
3
- class LoggedMail
4
- include DataMapper::Resource
5
-
6
- class << self
7
- attr_accessor :log_path, :log_processed
8
- end
9
-
10
- # Enabling this will save records for every processed email, not just the errored emails.
11
- self.log_processed = false
12
- self.log_path = File.join(Astrotrain.root, 'messages')
13
-
14
- property :id, Serial
15
- property :mapping_id, Integer, :index => true
16
- property :message_id, String, :index => true, :size => 255, :length => 1..255
17
- property :sender, String, :index => true, :size => 255, :length => 1..255
18
- property :recipient, String, :index => true, :size => 255, :length => 1..255
19
- property :subject, String, :index => true, :size => 255, :length => 1..255
20
- property :mail_file, String, :size => 255, :length => 1..255
21
- property :created_at, DateTime
22
- property :delivered_at, DateTime
23
- property :error_message, Text
24
-
25
- belongs_to :mapping
26
-
27
- def self.from(message, file = nil)
28
- logged = new
29
- begin
30
- logged.message_id = message.message_id
31
- logged.sender = Message.parse_email_addresses(message.sender).first
32
- logged.subject = message.subject
33
- logged.mail_file = file if file
34
- end
35
- if !block_given? || yield(logged)
36
- begin
37
- logged.save
38
- if logged.delivered_at && File.exist?(logged.mail_file.to_s)
39
- FileUtils.rm_rf logged.mail_file
40
- end
41
- rescue
42
- puts $!.inspect
43
- end
44
- end
45
- logged
46
- end
47
- end
48
- end
@@ -1,162 +0,0 @@
1
- module Astrotrain
2
- class Mapping
3
- include DataMapper::Resource
4
-
5
- class << self
6
- attr_accessor :default_domain
7
- attr_accessor :transports
8
- end
9
-
10
- self.transports = {"HTTP Post" => 'http_post', "Jabber" => 'jabber'}
11
- self.default_domain = 'astrotrain.com'
12
-
13
- property :id, Serial
14
- property :email_user, String, :size => 255, :length => 1..255, :index => :email, :format => /^[\w\.\_\%\+\-]*\*?$/
15
- property :email_domain, String, :size => 255, :lenght => 1..255, :index => :email, :format => /^[\w\-\_\.]+$/, :default => lambda { default_domain }
16
- property :destination, String, :size => 255, :length => 1..255
17
- property :transport, String, :size => 255, :set => transports.values, :default => 'http_post'
18
- property :separator, String, :size => 255
19
-
20
- validates_is_unique :email_user, :scope => :email_domain
21
- validates_format :destination, :as => /^(https?:)\/\/[^\/]+\/?/i, :if => :destination_uses_url?
22
- validates_format :destination, :as => :email_address, :if => :destination_uses_email?
23
-
24
- has n, :logged_mails, :order => [:created_at.desc]
25
-
26
- # returns a mapping for the given array of email addresses
27
- def self.match(email_addresses)
28
- email_addresses.each do |email_address|
29
- email_address.strip!
30
- email_address.downcase!
31
- name, domain = email_address.split("@")
32
- if mapping = match_by_address(name, domain) || match_by_wildcard(name, domain)
33
- return [mapping, email_address]
34
- end
35
- end
36
- nil
37
- end
38
-
39
- # Processes a given message. It finds a mapping, creates a LoggedMail record,
40
- # and attempts to process the message.
41
- def self.process(message, file = nil)
42
- LoggedMail.from(message, file) do |logged|
43
- save_logged = begin
44
- mapping, recipient = match(message.recipients)
45
- if mapping
46
- logged.recipient = recipient
47
- logged.mapping = mapping
48
- begin
49
- mapping.process(message, recipient)
50
- rescue Astrotrain::ProcessingCancelled
51
- logged.error_message = "Cancelled."
52
- end
53
- logged.delivered_at = Time.now.utc
54
- end
55
- LoggedMail.log_processed # save successfully processed messages?
56
- rescue
57
- logged.error_message = "#{$!.message}\n#{$!.backtrace.join("\n")}"
58
- end
59
- Astrotrain.callback(:post_processing, message, mapping, logged)
60
- save_logged
61
- end
62
- end
63
-
64
- # Processes a given message and recipient against the mapping's transport.
65
- def process(message, recipient)
66
- Astrotrain.callback(:pre_processing, message, self)
67
- Transport.process(message, self, recipient)
68
-
69
- end
70
-
71
- # returns true if the email matches this mapping. Wildcards in the name are allowed.
72
- # A mapping with foo*@bar.com will match foo@bar.com and food@bar.com, but not foo@baz.com.
73
- def match?(name, domain)
74
- email_domain == domain && name =~ email_user_regex
75
- end
76
-
77
- def destination_uses_url?
78
- transport == 'http_post'
79
- end
80
-
81
- def destination_uses_email?
82
- transport == 'jabber'
83
- end
84
-
85
- def full_email
86
- "#{email_user}@#{email_domain}"
87
- end
88
-
89
- # Looks for the mapping's separator in the message body and pulls only the content
90
- # above it. Assuming a separator of '===='...
91
- #
92
- # This will be kept
93
- #
94
- # On Thu, Sep 3, 2009 at 12:34 AM... (this will be removed)
95
- # ====
96
- #
97
- # > Everything here will be removed.
98
- #
99
- def find_reply_from(body)
100
- return if separator.blank?
101
- return '' if body.blank?
102
- lines = body.split("\n")
103
- delim_line = found_empty = nil
104
-
105
- (lines.size - 1).downto(0) do |i|
106
- line = lines[i]
107
- if !delim_line && line.include?(separator)
108
- delim_line = i
109
- elsif delim_line && !found_empty
110
- delim_line = i
111
- found_empty = line.strip.blank?
112
- elsif delim_line && found_empty
113
- if date_reply_line?(line) || line.strip.blank?
114
- delim_line = i
115
- else
116
- break
117
- end
118
- end
119
- end
120
-
121
- if delim_line
122
- body = if delim_line.zero?
123
- []
124
- elsif lines.size >= delim_line
125
- lines[0..delim_line-1]
126
- else
127
- lines
128
- end.join("\n")
129
- elsif body.frozen?
130
- body = body.dup
131
- end
132
- body.strip!
133
- body
134
- end
135
-
136
- protected
137
- def self.match_by_address(name, domain)
138
- first(:email_user => name, :email_domain => domain)
139
- end
140
-
141
- def self.match_by_wildcard(name, domain)
142
- wildcards = all(:email_domain => domain, :email_user.like => "%*")
143
- wildcards.sort! { |x, y| y.email_user.size <=> x.email_user.size }
144
- wildcards.detect { |w| w.match?(name, domain) }
145
- end
146
-
147
- DATE_LANGUATE_REGEXES = [/^on\b.*wrote\b?:$/i, /^am\b.*schrieb [\w\d\s]+:$/i, /^le\b.*a écrit\b?:$/i]
148
- def date_reply_line?(line)
149
- DATE_LANGUATE_REGEXES.any? { |re| line =~ re }
150
- end
151
-
152
- def email_user_regex
153
- @email_user_regex ||= begin
154
- if email_user['*']
155
- /^#{email_user.sub /\*/, '(.*)'}$/
156
- else
157
- /^#{email_user}$/
158
- end
159
- end
160
- end
161
- end
162
- end
@@ -1,18 +0,0 @@
1
- module Astrotrain
2
- class Mapping
3
- class HttpPost < Transport
4
- def process
5
- return unless Transport.processing
6
- RestClient.post @mapping.destination, fields.merge(:emails => fields[:emails].join(","))
7
- end
8
-
9
- def fields
10
- super
11
- @message.attachments.each_with_index do |att, index|
12
- @fields[:"attachments_#{index}"] = att
13
- end
14
- @fields
15
- end
16
- end
17
- end
18
- end
@@ -1,28 +0,0 @@
1
- begin
2
- require 'xmpp4r-simple'
3
- module Astrotrain
4
- class Mapping
5
- # This is experimental. No attachments are supported.
6
- class Jabber < Transport
7
- class << self
8
- attr_accessor :login, :password
9
- end
10
-
11
- def process
12
- return unless Transport.processing
13
- connection.deliver(@mapping.destination, content)
14
- end
15
-
16
- def connection
17
- @connection ||= ::Jabber::Simple.new(self.class.login, self.class.password)
18
- end
19
-
20
- def content
21
- @content ||= "From: %s\nTo: %s\nSubject: %s\nEmails: %s\n%s" % [fields[:from], fields[:to], fields[:subject], fields[:emails] * ", ", fields[:body]]
22
- end
23
- end
24
- end
25
- end
26
- rescue LoadError
27
- puts "Install xmpp4r-simple for Jabber support."
28
- end
@@ -1,55 +0,0 @@
1
- module Astrotrain
2
- class Mapping
3
- class Transport
4
- class << self
5
- attr_accessor :processing
6
- end
7
-
8
- # Enable this turn on processing.
9
- self.processing = false
10
-
11
- attr_reader :message, :mapping
12
-
13
- # process a given message against the mapping. The mapping transport is checked,
14
- # and the appropirate transport class handles the request.
15
- def self.process(message, mapping, recipient)
16
- case mapping.transport
17
- when 'http_post' then HttpPost.process(message, mapping, recipient)
18
- when 'jabber' then Jabber.process(message, mapping, recipient)
19
- end
20
- end
21
-
22
- def initialize(message, mapping, recipient)
23
- message.body = mapping.find_reply_from(message.body)
24
- @message = message
25
- @mapping = mapping
26
- @recipient = recipient
27
- end
28
-
29
- def process
30
- raise UnimplementedError
31
- end
32
-
33
- def fields
34
- @fields ||= begin
35
- all_emails = @message.recipients - [@recipient]
36
- f = {:subject => @message.subject, :to => @recipient, :from => @message.sender, :body => @message.body, :emails => all_emails, :html => @message.html}
37
- @message.headers.each do |key, value|
38
- f["headers[#{key}]"] = value
39
- end
40
- f
41
- end
42
- end
43
-
44
- # defines custom #process class methods that instantiate the class and calls a #process instance method
45
- def self.inherited(child)
46
- super
47
- class << child
48
- def process(message, mapping, recipient)
49
- new(message, mapping, recipient).process
50
- end
51
- end
52
- end
53
- end
54
- end
55
- end