ryespy 0.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +16 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +79 -0
- data/Rakefile +1 -0
- data/bin/ryespy.rb +187 -0
- data/lib/ryespy.rb +113 -0
- data/lib/ryespy/config.rb +76 -0
- data/lib/ryespy/listeners/ftp.rb +61 -0
- data/lib/ryespy/listeners/imap.rb +51 -0
- data/lib/ryespy/notifiers/sidekiq.rb +50 -0
- data/lib/ryespy/redis_conn.rb +32 -0
- data/lib/ryespy/version.rb +5 -0
- data/ryespy.gemspec +28 -0
- data/test/ryespy.rb +2 -0
- data/test/ryespy/config.rb +71 -0
- data/test/ryespy/version.rb +12 -0
- metadata +113 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: b26d8ba921285e533a4814b01fcffe3980eb932a
|
4
|
+
data.tar.gz: fc1df40db4c30717aa1598c1dd9af688971a6f39
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 16994fd3c32d506234d9c416e0e546dfbae89c9ae4c5ce8f87fcda96092b6761be593453dfe6372be46747ef26999f712b3ada6108af5979c390005b336e4302
|
7
|
+
data.tar.gz: 67d08264dc29abf677f9a5c46bbcfc0b27cb84e2a531085418cbb2acb1f136c2a1928b6c6cfdcb915305deb12c96fd7ce2b7fb3e51608c735feec1b767e8d181
|
data/.gitignore
ADDED
data/.ruby-gemset
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
ryespy
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
ruby-2.0.0-p0
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013 tiredpixel
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,79 @@
|
|
1
|
+
# Ryespy
|
2
|
+
|
3
|
+
Ryespy provides a simple executable for listening to IMAP mailboxes or FTP
|
4
|
+
folders, keeps track of what it's seen using Redis, and notifies Redis in a way
|
5
|
+
in which [Resque](https://github.com/resque/resque) and
|
6
|
+
[Sidekiq](https://github.com/mperham/sidekiq) can process using workers.
|
7
|
+
|
8
|
+
Ryespy was inspired by [Redimap](https://github.com/tiredpixel/redimap). Yes,
|
9
|
+
it's sometimes possible to inspire oneself. Ryespy with my little eye.
|
10
|
+
|
11
|
+
More sleep lost by [tiredpixel](http://www.tiredpixel.com).
|
12
|
+
|
13
|
+
|
14
|
+
## Installation
|
15
|
+
|
16
|
+
Install using:
|
17
|
+
|
18
|
+
$ gem install ryespy
|
19
|
+
|
20
|
+
|
21
|
+
## Usage
|
22
|
+
|
23
|
+
View the available options:
|
24
|
+
|
25
|
+
$ bundle exec ryespy --help
|
26
|
+
|
27
|
+
It is necessary to choose a listener (IMAP|FTP) and a notifier (Sidekiq).
|
28
|
+
|
29
|
+
Check IMAP and queue new emails and quit:
|
30
|
+
|
31
|
+
$ bundle exec ryespy --listener imap --imap-host mail.example.com --imap-username a@example.com --imap-password helpimacarrot --notifier-sidekiq
|
32
|
+
|
33
|
+
Check FTP and queue new files and quit:
|
34
|
+
|
35
|
+
$ bundle exec ryespy --listener ftp --ftp-host ftp.example.com --ftp-username b@example.com --ftp-password helpimacucumber --notifier-sidekiq
|
36
|
+
|
37
|
+
IMAP SSL and FTP PASSIVE are also supported. It's also possible to watch more
|
38
|
+
than one IMAP mailbox or FTP directory. The `--help` is most helpful.
|
39
|
+
|
40
|
+
Use `--eternal` to run eternally.
|
41
|
+
|
42
|
+
|
43
|
+
## Growing Like Flowers
|
44
|
+
|
45
|
+
Coming soon is a grand refactor sprinkled with lots of testing, ensuring that
|
46
|
+
present code is stable. Then, something or other else. Like a URL notifier.
|
47
|
+
Or maybe more listeners. Stay tuned -- or better still, help with the tuning.
|
48
|
+
|
49
|
+
|
50
|
+
## Contributions
|
51
|
+
|
52
|
+
Contributions are embraced with much love and affection! Please fork the
|
53
|
+
repository and wizard your magic, ensuring that any tests are not broken by the
|
54
|
+
changes. Then send a pull request. Simples! If you'd like to discuss what you're
|
55
|
+
doing or planning to do, or if you get stuck on something, then just wave. :)
|
56
|
+
|
57
|
+
Do whatever makes you happy. We'll probably still like you. :)
|
58
|
+
|
59
|
+
Tests are written using [minitest](https://github.com/seattlerb/minitest), which
|
60
|
+
is included by default in Ruby 1.9 onwards. To run all tests in a pretty way:
|
61
|
+
|
62
|
+
ruby -rminitest/pride test/ryespy.rb
|
63
|
+
|
64
|
+
Or, if you're of that turn of mind, use [TURN](https://github.com/TwP/turn)
|
65
|
+
(`gem install turn`):
|
66
|
+
|
67
|
+
turn test/ryespy.rb
|
68
|
+
|
69
|
+
|
70
|
+
## Blessing
|
71
|
+
|
72
|
+
May you find peace, and help others to do likewise.
|
73
|
+
|
74
|
+
|
75
|
+
## Licence
|
76
|
+
|
77
|
+
© [tiredpixel](http://www.tiredpixel.com) 2013. It is free software, released
|
78
|
+
under the MIT License, and may be redistributed under the terms specified in
|
79
|
+
`LICENSE`.
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
data/bin/ryespy.rb
ADDED
@@ -0,0 +1,187 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$stdout.sync = true
|
4
|
+
|
5
|
+
require 'optparse'
|
6
|
+
require 'ostruct'
|
7
|
+
|
8
|
+
require File.expand_path(File.dirname(__FILE__) + '/../lib/ryespy')
|
9
|
+
|
10
|
+
|
11
|
+
# = Parse opts
|
12
|
+
|
13
|
+
options = OpenStruct.new
|
14
|
+
|
15
|
+
options.notifiers = {
|
16
|
+
:sidekiq => [],
|
17
|
+
}
|
18
|
+
|
19
|
+
OptionParser.new do |opts|
|
20
|
+
opts.banner = "Usage: ryespy [options]"
|
21
|
+
|
22
|
+
opts.separator ""
|
23
|
+
opts.separator "Listener:"
|
24
|
+
|
25
|
+
opts.on("-l", "--listener LISTENER", [:imap, :ftp], "Listener (imap|ftp)") do |o|
|
26
|
+
options[:listener] = o
|
27
|
+
end
|
28
|
+
|
29
|
+
opts.separator ""
|
30
|
+
opts.separator "Polling:"
|
31
|
+
|
32
|
+
opts.on("-e", "--[no-]eternal", "Run eternally") do |o|
|
33
|
+
options[:eternal] = o
|
34
|
+
end
|
35
|
+
|
36
|
+
opts.on("--polling-interval [N]", Integer, "Poll every N seconds when --eternal") do |o|
|
37
|
+
options[:polling_interval] = o
|
38
|
+
end
|
39
|
+
|
40
|
+
opts.separator ""
|
41
|
+
opts.separator "Redis:"
|
42
|
+
|
43
|
+
opts.on("--redis-url [URL]", "Connect Redis to URL") do |o|
|
44
|
+
options[:redis_url] = o
|
45
|
+
end
|
46
|
+
|
47
|
+
opts.on("--redis-ns-ryespy [NS]", "Namespace Redis 'ryespy:' as NS") do |o|
|
48
|
+
options[:redis_ns_ryespy] = o
|
49
|
+
end
|
50
|
+
|
51
|
+
opts.separator ""
|
52
|
+
opts.separator "Listener imap:"
|
53
|
+
|
54
|
+
opts.on("--imap-host HOST", "Connect IMAP with HOST") do |o|
|
55
|
+
options[:imap_host] = o
|
56
|
+
end
|
57
|
+
|
58
|
+
opts.on("--imap-port [PORT]", Integer, "Connect IMAP with PORT") do |o|
|
59
|
+
options[:imap_port] = o
|
60
|
+
end
|
61
|
+
|
62
|
+
opts.on("--[no-]imap-ssl", "Connect IMAP using SSL") do |o|
|
63
|
+
options[:imap_ssl] = o
|
64
|
+
end
|
65
|
+
|
66
|
+
opts.on("--imap-username USERNAME", "Connect IMAP with USERNAME") do |o|
|
67
|
+
options[:imap_username] = o
|
68
|
+
end
|
69
|
+
|
70
|
+
opts.on("--imap-password PASSWORD", "Connect IMAP with PASSWORD") do |o|
|
71
|
+
options[:imap_password] = o
|
72
|
+
end
|
73
|
+
|
74
|
+
opts.on("--imap-mailboxes [INBOX,DEV]", Array, "Read IMAP MAILBOXES") do |o|
|
75
|
+
options[:imap_mailboxes] = o
|
76
|
+
end
|
77
|
+
|
78
|
+
opts.separator ""
|
79
|
+
opts.separator "Listener ftp:"
|
80
|
+
|
81
|
+
opts.on("--ftp-host HOST", "Connect FTP with HOST") do |o|
|
82
|
+
options[:ftp_host] = o
|
83
|
+
end
|
84
|
+
|
85
|
+
opts.on("--[no-]ftp-passive", "Connect FTP using PASSIVE mode") do |o|
|
86
|
+
options[:ftp_passive] = o
|
87
|
+
end
|
88
|
+
|
89
|
+
opts.on("--ftp-username USERNAME", "Connect FTP with USERNAME") do |o|
|
90
|
+
options[:ftp_username] = o
|
91
|
+
end
|
92
|
+
|
93
|
+
opts.on("--ftp-password PASSWORD", "Connect FTP with PASSWORD") do |o|
|
94
|
+
options[:ftp_password] = o
|
95
|
+
end
|
96
|
+
|
97
|
+
opts.on("--ftp-dirs [dir1,dir2]", Array, "Read FTP DIRS") do |o|
|
98
|
+
options[:ftp_dirs] = o
|
99
|
+
end
|
100
|
+
|
101
|
+
opts.separator ""
|
102
|
+
opts.separator "Notifier sidekiq:"
|
103
|
+
|
104
|
+
opts.on("--notifier-sidekiq [URL]", "Notify Sidekiq/Resque at Redis URL") do |o|
|
105
|
+
options.notifiers[:sidekiq] << o
|
106
|
+
end
|
107
|
+
|
108
|
+
opts.separator ""
|
109
|
+
opts.separator "Other:"
|
110
|
+
|
111
|
+
opts.on("-v", "--[no-]verbose", "Be somewhat verbose") do |o|
|
112
|
+
options[:verbose] = o
|
113
|
+
end
|
114
|
+
|
115
|
+
opts.on_tail("--help", "Show this message") do
|
116
|
+
puts opts
|
117
|
+
exit
|
118
|
+
end
|
119
|
+
|
120
|
+
opts.on_tail("--version", "Show version") do
|
121
|
+
puts "Ryespy version:#{Ryespy::VERSION}"
|
122
|
+
exit
|
123
|
+
end
|
124
|
+
end.parse!
|
125
|
+
|
126
|
+
[
|
127
|
+
:listener,
|
128
|
+
].each do |o|
|
129
|
+
unless options[o]
|
130
|
+
raise OptionParser::MissingArgument, "--#{o}"
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
|
135
|
+
# = Configure
|
136
|
+
|
137
|
+
Ryespy.configure do |c|
|
138
|
+
c.log_level = 'DEBUG' if options[:verbose]
|
139
|
+
|
140
|
+
c.listener = options[:listener]
|
141
|
+
|
142
|
+
params = [
|
143
|
+
:polling_interval,
|
144
|
+
:redis_url,
|
145
|
+
:redis_ns_ryespy,
|
146
|
+
:notifiers,
|
147
|
+
]
|
148
|
+
|
149
|
+
params.concat case c.listener
|
150
|
+
when :imap
|
151
|
+
[
|
152
|
+
:imap_host,
|
153
|
+
:imap_port,
|
154
|
+
:imap_ssl,
|
155
|
+
:imap_username,
|
156
|
+
:imap_password,
|
157
|
+
:imap_mailboxes,
|
158
|
+
]
|
159
|
+
when :ftp
|
160
|
+
[
|
161
|
+
:ftp_host,
|
162
|
+
:ftp_passive,
|
163
|
+
:ftp_username,
|
164
|
+
:ftp_password,
|
165
|
+
:ftp_dirs,
|
166
|
+
]
|
167
|
+
else
|
168
|
+
[]
|
169
|
+
end
|
170
|
+
|
171
|
+
params.each { |s| c.send("#{s}=", options[s]) unless options[s].nil? }
|
172
|
+
end
|
173
|
+
|
174
|
+
@logger = Ryespy.logger
|
175
|
+
|
176
|
+
|
177
|
+
# = Main loop
|
178
|
+
|
179
|
+
loop do
|
180
|
+
Ryespy.check_listener
|
181
|
+
|
182
|
+
break unless options[:eternal]
|
183
|
+
|
184
|
+
@logger.debug { "Snoring for #{Ryespy.config.polling_interval} s" }
|
185
|
+
|
186
|
+
sleep Ryespy.config.polling_interval # sleep awhile (snore)
|
187
|
+
end
|
data/lib/ryespy.rb
ADDED
@@ -0,0 +1,113 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
require_relative 'ryespy/version'
|
4
|
+
require_relative 'ryespy/config'
|
5
|
+
require_relative 'ryespy/redis_conn'
|
6
|
+
|
7
|
+
require_relative 'ryespy/listeners/imap'
|
8
|
+
require_relative 'ryespy/listeners/ftp'
|
9
|
+
|
10
|
+
require_relative 'ryespy/notifiers/sidekiq'
|
11
|
+
|
12
|
+
|
13
|
+
module Ryespy
|
14
|
+
|
15
|
+
extend self
|
16
|
+
|
17
|
+
def config
|
18
|
+
@config ||= Ryespy::Config.new
|
19
|
+
end
|
20
|
+
|
21
|
+
def configure
|
22
|
+
yield config
|
23
|
+
|
24
|
+
Ryespy.logger.debug { "Configured #{Ryespy.config.to_s}" }
|
25
|
+
end
|
26
|
+
|
27
|
+
def logger
|
28
|
+
unless @logger
|
29
|
+
@logger = Logger.new($stdout)
|
30
|
+
|
31
|
+
@logger.level = Logger.const_get(Ryespy.config.log_level)
|
32
|
+
end
|
33
|
+
|
34
|
+
@logger
|
35
|
+
end
|
36
|
+
|
37
|
+
def check_listener
|
38
|
+
redis_prefix = "#{Ryespy.config.redis_ns_ryespy}#{Ryespy.config.listener}:"
|
39
|
+
|
40
|
+
notifiers = []
|
41
|
+
|
42
|
+
begin
|
43
|
+
Ryespy.config.notifiers[:sidekiq].each do |notifier_instance|
|
44
|
+
notifiers << Ryespy::Notifier::Sidekiq.new(notifier_instance)
|
45
|
+
end
|
46
|
+
|
47
|
+
Ryespy::RedisConn.new(Ryespy.config.redis_url) do |redis|
|
48
|
+
Ryespy.send("check_#{Ryespy.config.listener}", redis, redis_prefix, notifiers)
|
49
|
+
end
|
50
|
+
ensure
|
51
|
+
notifiers.each { |n| n.close }
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def check_imap(redis, redis_prefix, notifiers)
|
56
|
+
redis_prefix += "#{Ryespy.config.imap_host},#{Ryespy.config.imap_port}:#{Ryespy.config.imap_username}:"
|
57
|
+
|
58
|
+
Ryespy::Listener::IMAP.new do |listener|
|
59
|
+
Ryespy.config.imap_mailboxes.each do |mailbox|
|
60
|
+
Ryespy.logger.debug { "mailbox:#{mailbox}" }
|
61
|
+
|
62
|
+
redis_key = redis_prefix + "#{mailbox}"
|
63
|
+
|
64
|
+
Ryespy.logger.debug { "redis_key:#{redis_key}" }
|
65
|
+
|
66
|
+
new_items = listener.check({
|
67
|
+
:mailbox => mailbox,
|
68
|
+
:last_seen_uid => redis.get(redis_key).to_i,
|
69
|
+
})
|
70
|
+
|
71
|
+
Ryespy.logger.debug { "new_items:#{new_items}" }
|
72
|
+
|
73
|
+
new_items.each do |uid|
|
74
|
+
redis.set(redis_key, uid)
|
75
|
+
|
76
|
+
notifiers.each { |n| n.notify('RyespyIMAPJob', [mailbox, uid]) }
|
77
|
+
end
|
78
|
+
|
79
|
+
Ryespy.logger.info { "#{mailbox} has #{new_items.count} new emails" }
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def check_ftp(redis, redis_prefix, notifiers)
|
85
|
+
redis_prefix += "#{Ryespy.config.ftp_host}:#{Ryespy.config.ftp_username}:"
|
86
|
+
|
87
|
+
Ryespy::Listener::FTP.new do |listener|
|
88
|
+
Ryespy.config.ftp_dirs.each do |dir|
|
89
|
+
Ryespy.logger.debug { "dir:#{dir}" }
|
90
|
+
|
91
|
+
redis_key = redis_prefix + "#{dir}"
|
92
|
+
|
93
|
+
Ryespy.logger.debug { "redis_key:#{redis_key}" }
|
94
|
+
|
95
|
+
new_items = listener.check({
|
96
|
+
:dir => dir,
|
97
|
+
:seen_files => redis.hgetall(redis_key),
|
98
|
+
})
|
99
|
+
|
100
|
+
Ryespy.logger.debug { "new_items:#{new_items}" }
|
101
|
+
|
102
|
+
new_items.each do |filename, checksum|
|
103
|
+
redis.hset(redis_key, filename, checksum)
|
104
|
+
|
105
|
+
notifiers.each { |n| n.notify('RyespyFTPJob', [dir, filename]) }
|
106
|
+
end
|
107
|
+
|
108
|
+
Ryespy.logger.info { "#{dir} has #{new_items.count} new files" }
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
module Ryespy
|
2
|
+
class Config
|
3
|
+
|
4
|
+
attr_accessor :log_level
|
5
|
+
attr_accessor :listener
|
6
|
+
attr_accessor :polling_interval
|
7
|
+
attr_accessor :redis_url
|
8
|
+
attr_accessor :redis_ns_ryespy
|
9
|
+
attr_accessor :notifiers
|
10
|
+
|
11
|
+
attr_accessor :imap_host
|
12
|
+
attr_accessor :imap_port
|
13
|
+
attr_accessor :imap_ssl
|
14
|
+
attr_accessor :imap_username
|
15
|
+
attr_accessor :imap_password
|
16
|
+
attr_accessor :imap_mailboxes
|
17
|
+
|
18
|
+
attr_accessor :ftp_host
|
19
|
+
attr_accessor :ftp_passive
|
20
|
+
attr_accessor :ftp_username
|
21
|
+
attr_accessor :ftp_password
|
22
|
+
attr_accessor :ftp_dirs
|
23
|
+
|
24
|
+
def initialize
|
25
|
+
@log_level = 'INFO'
|
26
|
+
@polling_interval = 60
|
27
|
+
@redis_ns_ryespy = 'ryespy:'
|
28
|
+
@notifiers = {
|
29
|
+
:sidekiq => [],
|
30
|
+
}
|
31
|
+
|
32
|
+
@imap_port = 993
|
33
|
+
@imap_ssl = true
|
34
|
+
@imap_mailboxes = ['INBOX']
|
35
|
+
|
36
|
+
@ftp_passive = false
|
37
|
+
@ftp_dirs = ['/']
|
38
|
+
end
|
39
|
+
|
40
|
+
def to_s
|
41
|
+
params = [
|
42
|
+
:log_level,
|
43
|
+
:listener,
|
44
|
+
:polling_interval,
|
45
|
+
:redis_url,
|
46
|
+
:redis_ns_ryespy,
|
47
|
+
:notifiers,
|
48
|
+
]
|
49
|
+
|
50
|
+
params.concat case @listener
|
51
|
+
when :imap
|
52
|
+
[
|
53
|
+
:imap_host,
|
54
|
+
:imap_port,
|
55
|
+
:imap_ssl,
|
56
|
+
:imap_username,
|
57
|
+
:imap_password,
|
58
|
+
]
|
59
|
+
when :ftp
|
60
|
+
[
|
61
|
+
:ftp_host,
|
62
|
+
:ftp_passive,
|
63
|
+
:ftp_username,
|
64
|
+
:ftp_dirs,
|
65
|
+
]
|
66
|
+
else
|
67
|
+
[]
|
68
|
+
end
|
69
|
+
|
70
|
+
params.collect! { |s| [s, instance_variable_get("@#{s}")] }
|
71
|
+
|
72
|
+
Hash[params].to_s
|
73
|
+
end
|
74
|
+
|
75
|
+
end
|
76
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
require 'net/ftp'
|
2
|
+
|
3
|
+
|
4
|
+
module Ryespy
|
5
|
+
module Listener
|
6
|
+
class FTP
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
begin
|
10
|
+
@ftp = Net::FTP.new(Ryespy.config.ftp_host)
|
11
|
+
|
12
|
+
@ftp.passive = Ryespy.config.ftp_passive
|
13
|
+
|
14
|
+
@ftp.login(Ryespy.config.ftp_username, Ryespy.config.ftp_password)
|
15
|
+
rescue Errno::ECONNREFUSED, Net::FTPError => e
|
16
|
+
Ryespy.logger.error { e.to_s }
|
17
|
+
|
18
|
+
return
|
19
|
+
end
|
20
|
+
|
21
|
+
if block_given?
|
22
|
+
yield self
|
23
|
+
|
24
|
+
close
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def close
|
29
|
+
@ftp.close
|
30
|
+
end
|
31
|
+
|
32
|
+
def check(params)
|
33
|
+
begin
|
34
|
+
@ftp.chdir(params[:dir])
|
35
|
+
|
36
|
+
objects = {}
|
37
|
+
|
38
|
+
@ftp.nlst.each do |fd|
|
39
|
+
mtime = @ftp.mtime(fd).to_i
|
40
|
+
size = @ftp.size(fd) rescue nil # ignore non-file error
|
41
|
+
|
42
|
+
if size # exclude directories
|
43
|
+
checksum = "#{mtime},#{size}"
|
44
|
+
|
45
|
+
if params[:seen_files][fd] != checksum
|
46
|
+
objects[fd] = checksum
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
objects
|
52
|
+
rescue Net::FTPError => e
|
53
|
+
Ryespy.logger.error { e.to_s }
|
54
|
+
|
55
|
+
return
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require 'net/imap'
|
2
|
+
|
3
|
+
|
4
|
+
module Ryespy
|
5
|
+
module Listener
|
6
|
+
class IMAP
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
begin
|
10
|
+
@imap = Net::IMAP.new(Ryespy.config.imap_host, {
|
11
|
+
:port => Ryespy.config.imap_port,
|
12
|
+
:ssl => Ryespy.config.imap_ssl,
|
13
|
+
})
|
14
|
+
|
15
|
+
@imap.login(Ryespy.config.imap_username, Ryespy.config.imap_password)
|
16
|
+
rescue Errno::ECONNREFUSED, Net::IMAP::Error => e
|
17
|
+
Ryespy.logger.error { e.to_s }
|
18
|
+
|
19
|
+
return
|
20
|
+
end
|
21
|
+
|
22
|
+
if block_given?
|
23
|
+
yield self
|
24
|
+
|
25
|
+
close
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def close
|
30
|
+
@imap.logout
|
31
|
+
|
32
|
+
@imap.disconnect
|
33
|
+
end
|
34
|
+
|
35
|
+
def check(params)
|
36
|
+
begin
|
37
|
+
@imap.select(params[:mailbox])
|
38
|
+
|
39
|
+
uids = @imap.uid_search("#{params[:last_seen_uid] + 1}:*")
|
40
|
+
|
41
|
+
uids.find_all { |uid| uid > params[:last_seen_uid] } # IMAP search gets fun with edge cases
|
42
|
+
rescue Net::IMAP::Error => e
|
43
|
+
Ryespy.logger.error { e.to_s }
|
44
|
+
|
45
|
+
return
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'securerandom'
|
3
|
+
|
4
|
+
require_relative '../redis_conn'
|
5
|
+
|
6
|
+
|
7
|
+
module Ryespy
|
8
|
+
module Notifier
|
9
|
+
class Sidekiq
|
10
|
+
|
11
|
+
REDIS_NS_RESQUE = 'resque'
|
12
|
+
RESQUE_QUEUE = 'ryespy'
|
13
|
+
|
14
|
+
def initialize(url = nil)
|
15
|
+
begin
|
16
|
+
@redis_conn = Ryespy::RedisConn.new(url)
|
17
|
+
rescue Errno::ECONNREFUSED, Net::FTPError => e
|
18
|
+
Ryespy.logger.error { e.to_s }
|
19
|
+
|
20
|
+
return
|
21
|
+
end
|
22
|
+
|
23
|
+
if block_given?
|
24
|
+
yield self
|
25
|
+
|
26
|
+
close
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def close
|
31
|
+
@redis_conn.close
|
32
|
+
end
|
33
|
+
|
34
|
+
def notify(job_class, args)
|
35
|
+
@redis_conn.redis.sadd("#{REDIS_NS_RESQUE}:queues", RESQUE_QUEUE)
|
36
|
+
|
37
|
+
@redis_conn.redis.rpush("#{REDIS_NS_RESQUE}:queue:#{RESQUE_QUEUE}", {
|
38
|
+
# resque
|
39
|
+
:class => job_class,
|
40
|
+
:args => args,
|
41
|
+
# sidekiq (extra)
|
42
|
+
:queue => RESQUE_QUEUE,
|
43
|
+
:retry => true,
|
44
|
+
:jid => SecureRandom.hex(12),
|
45
|
+
}.to_json)
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'redis'
|
2
|
+
|
3
|
+
|
4
|
+
module Ryespy
|
5
|
+
class RedisConn
|
6
|
+
|
7
|
+
attr_accessor :redis
|
8
|
+
|
9
|
+
def initialize(url = nil)
|
10
|
+
begin
|
11
|
+
@redis = Redis.connect(:url => url)
|
12
|
+
|
13
|
+
@redis.ping
|
14
|
+
rescue Redis::CannotConnectError => e
|
15
|
+
Ryespy.logger.error { e.to_s }
|
16
|
+
|
17
|
+
return
|
18
|
+
end
|
19
|
+
|
20
|
+
if block_given?
|
21
|
+
yield @redis
|
22
|
+
|
23
|
+
close
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def close
|
28
|
+
@redis.quit
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
end
|
data/ryespy.gemspec
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'ryespy/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "ryespy"
|
8
|
+
spec.version = Ryespy::VERSION
|
9
|
+
spec.authors = ["tiredpixel"]
|
10
|
+
spec.email = ["tp@tiredpixel.com"]
|
11
|
+
spec.description = %q{Ryespy provides a simple executable for listening to
|
12
|
+
IMAP mailboxes or FTP folders, keeps track of what it's seen using Redis,
|
13
|
+
and notifies Redis in a way in which Resque and Sidekiq can process using
|
14
|
+
workers.}
|
15
|
+
spec.summary = %q{Ryespy listens to IMAP and FTP and queues in Redis (Sidekiq/Resque).}
|
16
|
+
spec.homepage = "https://github.com/tiredpixel/ryespy"
|
17
|
+
spec.license = "MIT"
|
18
|
+
|
19
|
+
spec.files = `git ls-files`.split($/)
|
20
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
21
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
22
|
+
spec.require_paths = ["lib"]
|
23
|
+
|
24
|
+
spec.add_dependency "redis", "~> 3.0.4"
|
25
|
+
|
26
|
+
spec.add_development_dependency "bundler", "~> 1.3"
|
27
|
+
spec.add_development_dependency "rake"
|
28
|
+
end
|
data/test/ryespy.rb
ADDED
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'minitest/autorun'
|
2
|
+
|
3
|
+
require_relative '../../lib/ryespy'
|
4
|
+
require_relative '../../lib/ryespy/config'
|
5
|
+
|
6
|
+
|
7
|
+
describe Ryespy::Config do
|
8
|
+
|
9
|
+
describe "default" do
|
10
|
+
before do
|
11
|
+
@config = Ryespy::Config.new
|
12
|
+
end
|
13
|
+
|
14
|
+
it "sets log_level to INFO" do
|
15
|
+
@config.log_level.must_equal 'INFO'
|
16
|
+
end
|
17
|
+
|
18
|
+
it "sets polling_interval to 60" do
|
19
|
+
@config.polling_interval.must_equal 60
|
20
|
+
end
|
21
|
+
|
22
|
+
it "sets redis_ns_ryespy to ryespy:" do
|
23
|
+
@config.redis_ns_ryespy.must_equal 'ryespy:'
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
describe "configure block" do
|
28
|
+
before do
|
29
|
+
Ryespy.configure do |c|
|
30
|
+
c.log_level = 'ERROR'
|
31
|
+
c.listener = 'imap'
|
32
|
+
c.polling_interval = 13
|
33
|
+
c.redis_url = 'redis://127.0.0.1:6379/1'
|
34
|
+
c.redis_ns_ryespy = 'WithMyLittleEye!'
|
35
|
+
end
|
36
|
+
|
37
|
+
@config = Ryespy.config
|
38
|
+
end
|
39
|
+
|
40
|
+
it "configures log_level" do
|
41
|
+
@config.log_level.must_equal 'ERROR'
|
42
|
+
end
|
43
|
+
|
44
|
+
it "configures listener" do
|
45
|
+
@config.listener.must_equal 'imap'
|
46
|
+
end
|
47
|
+
|
48
|
+
it "configures polling_interval" do
|
49
|
+
@config.polling_interval.must_equal 13
|
50
|
+
end
|
51
|
+
|
52
|
+
it "configures redis_url" do
|
53
|
+
@config.redis_url.must_equal 'redis://127.0.0.1:6379/1'
|
54
|
+
end
|
55
|
+
|
56
|
+
it "configures redis_ns_ryespy" do
|
57
|
+
@config.redis_ns_ryespy.must_equal 'WithMyLittleEye!'
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
describe "#to_s" do
|
62
|
+
before do
|
63
|
+
@config = Ryespy::Config.new
|
64
|
+
end
|
65
|
+
|
66
|
+
it "stringifies hash of config" do
|
67
|
+
@config.to_s.must_equal '{:log_level=>"INFO", :listener=>nil, :polling_interval=>60, :redis_url=>nil, :redis_ns_ryespy=>"ryespy:", :notifiers=>{:sidekiq=>[]}}'
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
end
|
metadata
ADDED
@@ -0,0 +1,113 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: ryespy
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.6.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- tiredpixel
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-06-24 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: redis
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 3.0.4
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 3.0.4
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: bundler
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ~>
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.3'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ~>
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.3'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
description: |-
|
56
|
+
Ryespy provides a simple executable for listening to
|
57
|
+
IMAP mailboxes or FTP folders, keeps track of what it's seen using Redis,
|
58
|
+
and notifies Redis in a way in which Resque and Sidekiq can process using
|
59
|
+
workers.
|
60
|
+
email:
|
61
|
+
- tp@tiredpixel.com
|
62
|
+
executables:
|
63
|
+
- ryespy.rb
|
64
|
+
extensions: []
|
65
|
+
extra_rdoc_files: []
|
66
|
+
files:
|
67
|
+
- .gitignore
|
68
|
+
- .ruby-gemset
|
69
|
+
- .ruby-version
|
70
|
+
- Gemfile
|
71
|
+
- LICENSE.txt
|
72
|
+
- README.md
|
73
|
+
- Rakefile
|
74
|
+
- bin/ryespy.rb
|
75
|
+
- lib/ryespy.rb
|
76
|
+
- lib/ryespy/config.rb
|
77
|
+
- lib/ryespy/listeners/ftp.rb
|
78
|
+
- lib/ryespy/listeners/imap.rb
|
79
|
+
- lib/ryespy/notifiers/sidekiq.rb
|
80
|
+
- lib/ryespy/redis_conn.rb
|
81
|
+
- lib/ryespy/version.rb
|
82
|
+
- ryespy.gemspec
|
83
|
+
- test/ryespy.rb
|
84
|
+
- test/ryespy/config.rb
|
85
|
+
- test/ryespy/version.rb
|
86
|
+
homepage: https://github.com/tiredpixel/ryespy
|
87
|
+
licenses:
|
88
|
+
- MIT
|
89
|
+
metadata: {}
|
90
|
+
post_install_message:
|
91
|
+
rdoc_options: []
|
92
|
+
require_paths:
|
93
|
+
- lib
|
94
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
95
|
+
requirements:
|
96
|
+
- - '>='
|
97
|
+
- !ruby/object:Gem::Version
|
98
|
+
version: '0'
|
99
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - '>='
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
requirements: []
|
105
|
+
rubyforge_project:
|
106
|
+
rubygems_version: 2.0.3
|
107
|
+
signing_key:
|
108
|
+
specification_version: 4
|
109
|
+
summary: Ryespy listens to IMAP and FTP and queues in Redis (Sidekiq/Resque).
|
110
|
+
test_files:
|
111
|
+
- test/ryespy.rb
|
112
|
+
- test/ryespy/config.rb
|
113
|
+
- test/ryespy/version.rb
|