ryespy 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +16 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +79 -0
- data/Rakefile +1 -0
- data/bin/ryespy.rb +187 -0
- data/lib/ryespy.rb +113 -0
- data/lib/ryespy/config.rb +76 -0
- data/lib/ryespy/listeners/ftp.rb +61 -0
- data/lib/ryespy/listeners/imap.rb +51 -0
- data/lib/ryespy/notifiers/sidekiq.rb +50 -0
- data/lib/ryespy/redis_conn.rb +32 -0
- data/lib/ryespy/version.rb +5 -0
- data/ryespy.gemspec +28 -0
- data/test/ryespy.rb +2 -0
- data/test/ryespy/config.rb +71 -0
- data/test/ryespy/version.rb +12 -0
- metadata +113 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: b26d8ba921285e533a4814b01fcffe3980eb932a
|
4
|
+
data.tar.gz: fc1df40db4c30717aa1598c1dd9af688971a6f39
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 16994fd3c32d506234d9c416e0e546dfbae89c9ae4c5ce8f87fcda96092b6761be593453dfe6372be46747ef26999f712b3ada6108af5979c390005b336e4302
|
7
|
+
data.tar.gz: 67d08264dc29abf677f9a5c46bbcfc0b27cb84e2a531085418cbb2acb1f136c2a1928b6c6cfdcb915305deb12c96fd7ce2b7fb3e51608c735feec1b767e8d181
|
data/.gitignore
ADDED
data/.ruby-gemset
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
ryespy
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
ruby-2.0.0-p0
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013 tiredpixel
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,79 @@
|
|
1
|
+
# Ryespy
|
2
|
+
|
3
|
+
Ryespy provides a simple executable for listening to IMAP mailboxes or FTP
|
4
|
+
folders, keeps track of what it's seen using Redis, and notifies Redis in a way
|
5
|
+
in which [Resque](https://github.com/resque/resque) and
|
6
|
+
[Sidekiq](https://github.com/mperham/sidekiq) can process using workers.
|
7
|
+
|
8
|
+
Ryespy was inspired by [Redimap](https://github.com/tiredpixel/redimap). Yes,
|
9
|
+
it's sometimes possible to inspire oneself. Ryespy with my little eye.
|
10
|
+
|
11
|
+
More sleep lost by [tiredpixel](http://www.tiredpixel.com).
|
12
|
+
|
13
|
+
|
14
|
+
## Installation
|
15
|
+
|
16
|
+
Install using:
|
17
|
+
|
18
|
+
$ gem install ryespy
|
19
|
+
|
20
|
+
|
21
|
+
## Usage
|
22
|
+
|
23
|
+
View the available options:
|
24
|
+
|
25
|
+
$ bundle exec ryespy --help
|
26
|
+
|
27
|
+
It is necessary to choose a listener (IMAP|FTP) and a notifier (Sidekiq).
|
28
|
+
|
29
|
+
Check IMAP and queue new emails and quit:
|
30
|
+
|
31
|
+
$ bundle exec ryespy --listener imap --imap-host mail.example.com --imap-username a@example.com --imap-password helpimacarrot --notifier-sidekiq
|
32
|
+
|
33
|
+
Check FTP and queue new files and quit:
|
34
|
+
|
35
|
+
$ bundle exec ryespy --listener ftp --ftp-host ftp.example.com --ftp-username b@example.com --ftp-password helpimacucumber --notifier-sidekiq
|
36
|
+
|
37
|
+
IMAP SSL and FTP PASSIVE are also supported. It's also possible to watch more
|
38
|
+
than one IMAP mailbox or FTP directory. The `--help` is most helpful.
|
39
|
+
|
40
|
+
Use `--eternal` to run eternally.
|
41
|
+
|
42
|
+
|
43
|
+
## Growing Like Flowers
|
44
|
+
|
45
|
+
Coming soon is a grand refactor sprinkled with lots of testing, ensuring that
|
46
|
+
present code is stable. Then, something or other else. Like a URL notifier.
|
47
|
+
Or maybe more listeners. Stay tuned -- or better still, help with the tuning.
|
48
|
+
|
49
|
+
|
50
|
+
## Contributions
|
51
|
+
|
52
|
+
Contributions are embraced with much love and affection! Please fork the
|
53
|
+
repository and wizard your magic, ensuring that any tests are not broken by the
|
54
|
+
changes. Then send a pull request. Simples! If you'd like to discuss what you're
|
55
|
+
doing or planning to do, or if you get stuck on something, then just wave. :)
|
56
|
+
|
57
|
+
Do whatever makes you happy. We'll probably still like you. :)
|
58
|
+
|
59
|
+
Tests are written using [minitest](https://github.com/seattlerb/minitest), which
|
60
|
+
is included by default in Ruby 1.9 onwards. To run all tests in a pretty way:
|
61
|
+
|
62
|
+
ruby -rminitest/pride test/ryespy.rb
|
63
|
+
|
64
|
+
Or, if you're of that turn of mind, use [TURN](https://github.com/TwP/turn)
|
65
|
+
(`gem install turn`):
|
66
|
+
|
67
|
+
turn test/ryespy.rb
|
68
|
+
|
69
|
+
|
70
|
+
## Blessing
|
71
|
+
|
72
|
+
May you find peace, and help others to do likewise.
|
73
|
+
|
74
|
+
|
75
|
+
## Licence
|
76
|
+
|
77
|
+
© [tiredpixel](http://www.tiredpixel.com) 2013. It is free software, released
|
78
|
+
under the MIT License, and may be redistributed under the terms specified in
|
79
|
+
`LICENSE`.
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
data/bin/ryespy.rb
ADDED
@@ -0,0 +1,187 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$stdout.sync = true
|
4
|
+
|
5
|
+
require 'optparse'
|
6
|
+
require 'ostruct'
|
7
|
+
|
8
|
+
require File.expand_path(File.dirname(__FILE__) + '/../lib/ryespy')
|
9
|
+
|
10
|
+
|
11
|
+
# = Parse opts
|
12
|
+
|
13
|
+
options = OpenStruct.new
|
14
|
+
|
15
|
+
options.notifiers = {
|
16
|
+
:sidekiq => [],
|
17
|
+
}
|
18
|
+
|
19
|
+
OptionParser.new do |opts|
|
20
|
+
opts.banner = "Usage: ryespy [options]"
|
21
|
+
|
22
|
+
opts.separator ""
|
23
|
+
opts.separator "Listener:"
|
24
|
+
|
25
|
+
opts.on("-l", "--listener LISTENER", [:imap, :ftp], "Listener (imap|ftp)") do |o|
|
26
|
+
options[:listener] = o
|
27
|
+
end
|
28
|
+
|
29
|
+
opts.separator ""
|
30
|
+
opts.separator "Polling:"
|
31
|
+
|
32
|
+
opts.on("-e", "--[no-]eternal", "Run eternally") do |o|
|
33
|
+
options[:eternal] = o
|
34
|
+
end
|
35
|
+
|
36
|
+
opts.on("--polling-interval [N]", Integer, "Poll every N seconds when --eternal") do |o|
|
37
|
+
options[:polling_interval] = o
|
38
|
+
end
|
39
|
+
|
40
|
+
opts.separator ""
|
41
|
+
opts.separator "Redis:"
|
42
|
+
|
43
|
+
opts.on("--redis-url [URL]", "Connect Redis to URL") do |o|
|
44
|
+
options[:redis_url] = o
|
45
|
+
end
|
46
|
+
|
47
|
+
opts.on("--redis-ns-ryespy [NS]", "Namespace Redis 'ryespy:' as NS") do |o|
|
48
|
+
options[:redis_ns_ryespy] = o
|
49
|
+
end
|
50
|
+
|
51
|
+
opts.separator ""
|
52
|
+
opts.separator "Listener imap:"
|
53
|
+
|
54
|
+
opts.on("--imap-host HOST", "Connect IMAP with HOST") do |o|
|
55
|
+
options[:imap_host] = o
|
56
|
+
end
|
57
|
+
|
58
|
+
opts.on("--imap-port [PORT]", Integer, "Connect IMAP with PORT") do |o|
|
59
|
+
options[:imap_port] = o
|
60
|
+
end
|
61
|
+
|
62
|
+
opts.on("--[no-]imap-ssl", "Connect IMAP using SSL") do |o|
|
63
|
+
options[:imap_ssl] = o
|
64
|
+
end
|
65
|
+
|
66
|
+
opts.on("--imap-username USERNAME", "Connect IMAP with USERNAME") do |o|
|
67
|
+
options[:imap_username] = o
|
68
|
+
end
|
69
|
+
|
70
|
+
opts.on("--imap-password PASSWORD", "Connect IMAP with PASSWORD") do |o|
|
71
|
+
options[:imap_password] = o
|
72
|
+
end
|
73
|
+
|
74
|
+
opts.on("--imap-mailboxes [INBOX,DEV]", Array, "Read IMAP MAILBOXES") do |o|
|
75
|
+
options[:imap_mailboxes] = o
|
76
|
+
end
|
77
|
+
|
78
|
+
opts.separator ""
|
79
|
+
opts.separator "Listener ftp:"
|
80
|
+
|
81
|
+
opts.on("--ftp-host HOST", "Connect FTP with HOST") do |o|
|
82
|
+
options[:ftp_host] = o
|
83
|
+
end
|
84
|
+
|
85
|
+
opts.on("--[no-]ftp-passive", "Connect FTP using PASSIVE mode") do |o|
|
86
|
+
options[:ftp_passive] = o
|
87
|
+
end
|
88
|
+
|
89
|
+
opts.on("--ftp-username USERNAME", "Connect FTP with USERNAME") do |o|
|
90
|
+
options[:ftp_username] = o
|
91
|
+
end
|
92
|
+
|
93
|
+
opts.on("--ftp-password PASSWORD", "Connect FTP with PASSWORD") do |o|
|
94
|
+
options[:ftp_password] = o
|
95
|
+
end
|
96
|
+
|
97
|
+
opts.on("--ftp-dirs [dir1,dir2]", Array, "Read FTP DIRS") do |o|
|
98
|
+
options[:ftp_dirs] = o
|
99
|
+
end
|
100
|
+
|
101
|
+
opts.separator ""
|
102
|
+
opts.separator "Notifier sidekiq:"
|
103
|
+
|
104
|
+
opts.on("--notifier-sidekiq [URL]", "Notify Sidekiq/Resque at Redis URL") do |o|
|
105
|
+
options.notifiers[:sidekiq] << o
|
106
|
+
end
|
107
|
+
|
108
|
+
opts.separator ""
|
109
|
+
opts.separator "Other:"
|
110
|
+
|
111
|
+
opts.on("-v", "--[no-]verbose", "Be somewhat verbose") do |o|
|
112
|
+
options[:verbose] = o
|
113
|
+
end
|
114
|
+
|
115
|
+
opts.on_tail("--help", "Show this message") do
|
116
|
+
puts opts
|
117
|
+
exit
|
118
|
+
end
|
119
|
+
|
120
|
+
opts.on_tail("--version", "Show version") do
|
121
|
+
puts "Ryespy version:#{Ryespy::VERSION}"
|
122
|
+
exit
|
123
|
+
end
|
124
|
+
end.parse!
|
125
|
+
|
126
|
+
[
|
127
|
+
:listener,
|
128
|
+
].each do |o|
|
129
|
+
unless options[o]
|
130
|
+
raise OptionParser::MissingArgument, "--#{o}"
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
|
135
|
+
# = Configure
|
136
|
+
|
137
|
+
Ryespy.configure do |c|
|
138
|
+
c.log_level = 'DEBUG' if options[:verbose]
|
139
|
+
|
140
|
+
c.listener = options[:listener]
|
141
|
+
|
142
|
+
params = [
|
143
|
+
:polling_interval,
|
144
|
+
:redis_url,
|
145
|
+
:redis_ns_ryespy,
|
146
|
+
:notifiers,
|
147
|
+
]
|
148
|
+
|
149
|
+
params.concat case c.listener
|
150
|
+
when :imap
|
151
|
+
[
|
152
|
+
:imap_host,
|
153
|
+
:imap_port,
|
154
|
+
:imap_ssl,
|
155
|
+
:imap_username,
|
156
|
+
:imap_password,
|
157
|
+
:imap_mailboxes,
|
158
|
+
]
|
159
|
+
when :ftp
|
160
|
+
[
|
161
|
+
:ftp_host,
|
162
|
+
:ftp_passive,
|
163
|
+
:ftp_username,
|
164
|
+
:ftp_password,
|
165
|
+
:ftp_dirs,
|
166
|
+
]
|
167
|
+
else
|
168
|
+
[]
|
169
|
+
end
|
170
|
+
|
171
|
+
params.each { |s| c.send("#{s}=", options[s]) unless options[s].nil? }
|
172
|
+
end
|
173
|
+
|
174
|
+
@logger = Ryespy.logger
|
175
|
+
|
176
|
+
|
177
|
+
# = Main loop
|
178
|
+
|
179
|
+
loop do
|
180
|
+
Ryespy.check_listener
|
181
|
+
|
182
|
+
break unless options[:eternal]
|
183
|
+
|
184
|
+
@logger.debug { "Snoring for #{Ryespy.config.polling_interval} s" }
|
185
|
+
|
186
|
+
sleep Ryespy.config.polling_interval # sleep awhile (snore)
|
187
|
+
end
|
data/lib/ryespy.rb
ADDED
@@ -0,0 +1,113 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
require_relative 'ryespy/version'
|
4
|
+
require_relative 'ryespy/config'
|
5
|
+
require_relative 'ryespy/redis_conn'
|
6
|
+
|
7
|
+
require_relative 'ryespy/listeners/imap'
|
8
|
+
require_relative 'ryespy/listeners/ftp'
|
9
|
+
|
10
|
+
require_relative 'ryespy/notifiers/sidekiq'
|
11
|
+
|
12
|
+
|
13
|
+
module Ryespy
|
14
|
+
|
15
|
+
extend self
|
16
|
+
|
17
|
+
def config
|
18
|
+
@config ||= Ryespy::Config.new
|
19
|
+
end
|
20
|
+
|
21
|
+
def configure
|
22
|
+
yield config
|
23
|
+
|
24
|
+
Ryespy.logger.debug { "Configured #{Ryespy.config.to_s}" }
|
25
|
+
end
|
26
|
+
|
27
|
+
def logger
|
28
|
+
unless @logger
|
29
|
+
@logger = Logger.new($stdout)
|
30
|
+
|
31
|
+
@logger.level = Logger.const_get(Ryespy.config.log_level)
|
32
|
+
end
|
33
|
+
|
34
|
+
@logger
|
35
|
+
end
|
36
|
+
|
37
|
+
def check_listener
|
38
|
+
redis_prefix = "#{Ryespy.config.redis_ns_ryespy}#{Ryespy.config.listener}:"
|
39
|
+
|
40
|
+
notifiers = []
|
41
|
+
|
42
|
+
begin
|
43
|
+
Ryespy.config.notifiers[:sidekiq].each do |notifier_instance|
|
44
|
+
notifiers << Ryespy::Notifier::Sidekiq.new(notifier_instance)
|
45
|
+
end
|
46
|
+
|
47
|
+
Ryespy::RedisConn.new(Ryespy.config.redis_url) do |redis|
|
48
|
+
Ryespy.send("check_#{Ryespy.config.listener}", redis, redis_prefix, notifiers)
|
49
|
+
end
|
50
|
+
ensure
|
51
|
+
notifiers.each { |n| n.close }
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def check_imap(redis, redis_prefix, notifiers)
|
56
|
+
redis_prefix += "#{Ryespy.config.imap_host},#{Ryespy.config.imap_port}:#{Ryespy.config.imap_username}:"
|
57
|
+
|
58
|
+
Ryespy::Listener::IMAP.new do |listener|
|
59
|
+
Ryespy.config.imap_mailboxes.each do |mailbox|
|
60
|
+
Ryespy.logger.debug { "mailbox:#{mailbox}" }
|
61
|
+
|
62
|
+
redis_key = redis_prefix + "#{mailbox}"
|
63
|
+
|
64
|
+
Ryespy.logger.debug { "redis_key:#{redis_key}" }
|
65
|
+
|
66
|
+
new_items = listener.check({
|
67
|
+
:mailbox => mailbox,
|
68
|
+
:last_seen_uid => redis.get(redis_key).to_i,
|
69
|
+
})
|
70
|
+
|
71
|
+
Ryespy.logger.debug { "new_items:#{new_items}" }
|
72
|
+
|
73
|
+
new_items.each do |uid|
|
74
|
+
redis.set(redis_key, uid)
|
75
|
+
|
76
|
+
notifiers.each { |n| n.notify('RyespyIMAPJob', [mailbox, uid]) }
|
77
|
+
end
|
78
|
+
|
79
|
+
Ryespy.logger.info { "#{mailbox} has #{new_items.count} new emails" }
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def check_ftp(redis, redis_prefix, notifiers)
|
85
|
+
redis_prefix += "#{Ryespy.config.ftp_host}:#{Ryespy.config.ftp_username}:"
|
86
|
+
|
87
|
+
Ryespy::Listener::FTP.new do |listener|
|
88
|
+
Ryespy.config.ftp_dirs.each do |dir|
|
89
|
+
Ryespy.logger.debug { "dir:#{dir}" }
|
90
|
+
|
91
|
+
redis_key = redis_prefix + "#{dir}"
|
92
|
+
|
93
|
+
Ryespy.logger.debug { "redis_key:#{redis_key}" }
|
94
|
+
|
95
|
+
new_items = listener.check({
|
96
|
+
:dir => dir,
|
97
|
+
:seen_files => redis.hgetall(redis_key),
|
98
|
+
})
|
99
|
+
|
100
|
+
Ryespy.logger.debug { "new_items:#{new_items}" }
|
101
|
+
|
102
|
+
new_items.each do |filename, checksum|
|
103
|
+
redis.hset(redis_key, filename, checksum)
|
104
|
+
|
105
|
+
notifiers.each { |n| n.notify('RyespyFTPJob', [dir, filename]) }
|
106
|
+
end
|
107
|
+
|
108
|
+
Ryespy.logger.info { "#{dir} has #{new_items.count} new files" }
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
module Ryespy
|
2
|
+
class Config
|
3
|
+
|
4
|
+
attr_accessor :log_level
|
5
|
+
attr_accessor :listener
|
6
|
+
attr_accessor :polling_interval
|
7
|
+
attr_accessor :redis_url
|
8
|
+
attr_accessor :redis_ns_ryespy
|
9
|
+
attr_accessor :notifiers
|
10
|
+
|
11
|
+
attr_accessor :imap_host
|
12
|
+
attr_accessor :imap_port
|
13
|
+
attr_accessor :imap_ssl
|
14
|
+
attr_accessor :imap_username
|
15
|
+
attr_accessor :imap_password
|
16
|
+
attr_accessor :imap_mailboxes
|
17
|
+
|
18
|
+
attr_accessor :ftp_host
|
19
|
+
attr_accessor :ftp_passive
|
20
|
+
attr_accessor :ftp_username
|
21
|
+
attr_accessor :ftp_password
|
22
|
+
attr_accessor :ftp_dirs
|
23
|
+
|
24
|
+
def initialize
|
25
|
+
@log_level = 'INFO'
|
26
|
+
@polling_interval = 60
|
27
|
+
@redis_ns_ryespy = 'ryespy:'
|
28
|
+
@notifiers = {
|
29
|
+
:sidekiq => [],
|
30
|
+
}
|
31
|
+
|
32
|
+
@imap_port = 993
|
33
|
+
@imap_ssl = true
|
34
|
+
@imap_mailboxes = ['INBOX']
|
35
|
+
|
36
|
+
@ftp_passive = false
|
37
|
+
@ftp_dirs = ['/']
|
38
|
+
end
|
39
|
+
|
40
|
+
def to_s
|
41
|
+
params = [
|
42
|
+
:log_level,
|
43
|
+
:listener,
|
44
|
+
:polling_interval,
|
45
|
+
:redis_url,
|
46
|
+
:redis_ns_ryespy,
|
47
|
+
:notifiers,
|
48
|
+
]
|
49
|
+
|
50
|
+
params.concat case @listener
|
51
|
+
when :imap
|
52
|
+
[
|
53
|
+
:imap_host,
|
54
|
+
:imap_port,
|
55
|
+
:imap_ssl,
|
56
|
+
:imap_username,
|
57
|
+
:imap_password,
|
58
|
+
]
|
59
|
+
when :ftp
|
60
|
+
[
|
61
|
+
:ftp_host,
|
62
|
+
:ftp_passive,
|
63
|
+
:ftp_username,
|
64
|
+
:ftp_dirs,
|
65
|
+
]
|
66
|
+
else
|
67
|
+
[]
|
68
|
+
end
|
69
|
+
|
70
|
+
params.collect! { |s| [s, instance_variable_get("@#{s}")] }
|
71
|
+
|
72
|
+
Hash[params].to_s
|
73
|
+
end
|
74
|
+
|
75
|
+
end
|
76
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
require 'net/ftp'
|
2
|
+
|
3
|
+
|
4
|
+
module Ryespy
|
5
|
+
module Listener
|
6
|
+
class FTP
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
begin
|
10
|
+
@ftp = Net::FTP.new(Ryespy.config.ftp_host)
|
11
|
+
|
12
|
+
@ftp.passive = Ryespy.config.ftp_passive
|
13
|
+
|
14
|
+
@ftp.login(Ryespy.config.ftp_username, Ryespy.config.ftp_password)
|
15
|
+
rescue Errno::ECONNREFUSED, Net::FTPError => e
|
16
|
+
Ryespy.logger.error { e.to_s }
|
17
|
+
|
18
|
+
return
|
19
|
+
end
|
20
|
+
|
21
|
+
if block_given?
|
22
|
+
yield self
|
23
|
+
|
24
|
+
close
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def close
|
29
|
+
@ftp.close
|
30
|
+
end
|
31
|
+
|
32
|
+
def check(params)
|
33
|
+
begin
|
34
|
+
@ftp.chdir(params[:dir])
|
35
|
+
|
36
|
+
objects = {}
|
37
|
+
|
38
|
+
@ftp.nlst.each do |fd|
|
39
|
+
mtime = @ftp.mtime(fd).to_i
|
40
|
+
size = @ftp.size(fd) rescue nil # ignore non-file error
|
41
|
+
|
42
|
+
if size # exclude directories
|
43
|
+
checksum = "#{mtime},#{size}"
|
44
|
+
|
45
|
+
if params[:seen_files][fd] != checksum
|
46
|
+
objects[fd] = checksum
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
objects
|
52
|
+
rescue Net::FTPError => e
|
53
|
+
Ryespy.logger.error { e.to_s }
|
54
|
+
|
55
|
+
return
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require 'net/imap'
|
2
|
+
|
3
|
+
|
4
|
+
module Ryespy
|
5
|
+
module Listener
|
6
|
+
class IMAP
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
begin
|
10
|
+
@imap = Net::IMAP.new(Ryespy.config.imap_host, {
|
11
|
+
:port => Ryespy.config.imap_port,
|
12
|
+
:ssl => Ryespy.config.imap_ssl,
|
13
|
+
})
|
14
|
+
|
15
|
+
@imap.login(Ryespy.config.imap_username, Ryespy.config.imap_password)
|
16
|
+
rescue Errno::ECONNREFUSED, Net::IMAP::Error => e
|
17
|
+
Ryespy.logger.error { e.to_s }
|
18
|
+
|
19
|
+
return
|
20
|
+
end
|
21
|
+
|
22
|
+
if block_given?
|
23
|
+
yield self
|
24
|
+
|
25
|
+
close
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def close
|
30
|
+
@imap.logout
|
31
|
+
|
32
|
+
@imap.disconnect
|
33
|
+
end
|
34
|
+
|
35
|
+
def check(params)
|
36
|
+
begin
|
37
|
+
@imap.select(params[:mailbox])
|
38
|
+
|
39
|
+
uids = @imap.uid_search("#{params[:last_seen_uid] + 1}:*")
|
40
|
+
|
41
|
+
uids.find_all { |uid| uid > params[:last_seen_uid] } # IMAP search gets fun with edge cases
|
42
|
+
rescue Net::IMAP::Error => e
|
43
|
+
Ryespy.logger.error { e.to_s }
|
44
|
+
|
45
|
+
return
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'securerandom'
|
3
|
+
|
4
|
+
require_relative '../redis_conn'
|
5
|
+
|
6
|
+
|
7
|
+
module Ryespy
|
8
|
+
module Notifier
|
9
|
+
class Sidekiq
|
10
|
+
|
11
|
+
REDIS_NS_RESQUE = 'resque'
|
12
|
+
RESQUE_QUEUE = 'ryespy'
|
13
|
+
|
14
|
+
def initialize(url = nil)
|
15
|
+
begin
|
16
|
+
@redis_conn = Ryespy::RedisConn.new(url)
|
17
|
+
rescue Errno::ECONNREFUSED, Net::FTPError => e
|
18
|
+
Ryespy.logger.error { e.to_s }
|
19
|
+
|
20
|
+
return
|
21
|
+
end
|
22
|
+
|
23
|
+
if block_given?
|
24
|
+
yield self
|
25
|
+
|
26
|
+
close
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def close
|
31
|
+
@redis_conn.close
|
32
|
+
end
|
33
|
+
|
34
|
+
def notify(job_class, args)
|
35
|
+
@redis_conn.redis.sadd("#{REDIS_NS_RESQUE}:queues", RESQUE_QUEUE)
|
36
|
+
|
37
|
+
@redis_conn.redis.rpush("#{REDIS_NS_RESQUE}:queue:#{RESQUE_QUEUE}", {
|
38
|
+
# resque
|
39
|
+
:class => job_class,
|
40
|
+
:args => args,
|
41
|
+
# sidekiq (extra)
|
42
|
+
:queue => RESQUE_QUEUE,
|
43
|
+
:retry => true,
|
44
|
+
:jid => SecureRandom.hex(12),
|
45
|
+
}.to_json)
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'redis'
|
2
|
+
|
3
|
+
|
4
|
+
module Ryespy
|
5
|
+
class RedisConn
|
6
|
+
|
7
|
+
attr_accessor :redis
|
8
|
+
|
9
|
+
def initialize(url = nil)
|
10
|
+
begin
|
11
|
+
@redis = Redis.connect(:url => url)
|
12
|
+
|
13
|
+
@redis.ping
|
14
|
+
rescue Redis::CannotConnectError => e
|
15
|
+
Ryespy.logger.error { e.to_s }
|
16
|
+
|
17
|
+
return
|
18
|
+
end
|
19
|
+
|
20
|
+
if block_given?
|
21
|
+
yield @redis
|
22
|
+
|
23
|
+
close
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def close
|
28
|
+
@redis.quit
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
end
|
data/ryespy.gemspec
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'ryespy/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "ryespy"
|
8
|
+
spec.version = Ryespy::VERSION
|
9
|
+
spec.authors = ["tiredpixel"]
|
10
|
+
spec.email = ["tp@tiredpixel.com"]
|
11
|
+
spec.description = %q{Ryespy provides a simple executable for listening to
|
12
|
+
IMAP mailboxes or FTP folders, keeps track of what it's seen using Redis,
|
13
|
+
and notifies Redis in a way in which Resque and Sidekiq can process using
|
14
|
+
workers.}
|
15
|
+
spec.summary = %q{Ryespy listens to IMAP and FTP and queues in Redis (Sidekiq/Resque).}
|
16
|
+
spec.homepage = "https://github.com/tiredpixel/ryespy"
|
17
|
+
spec.license = "MIT"
|
18
|
+
|
19
|
+
spec.files = `git ls-files`.split($/)
|
20
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
21
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
22
|
+
spec.require_paths = ["lib"]
|
23
|
+
|
24
|
+
spec.add_dependency "redis", "~> 3.0.4"
|
25
|
+
|
26
|
+
spec.add_development_dependency "bundler", "~> 1.3"
|
27
|
+
spec.add_development_dependency "rake"
|
28
|
+
end
|
data/test/ryespy.rb
ADDED
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'minitest/autorun'
|
2
|
+
|
3
|
+
require_relative '../../lib/ryespy'
|
4
|
+
require_relative '../../lib/ryespy/config'
|
5
|
+
|
6
|
+
|
7
|
+
describe Ryespy::Config do
|
8
|
+
|
9
|
+
describe "default" do
|
10
|
+
before do
|
11
|
+
@config = Ryespy::Config.new
|
12
|
+
end
|
13
|
+
|
14
|
+
it "sets log_level to INFO" do
|
15
|
+
@config.log_level.must_equal 'INFO'
|
16
|
+
end
|
17
|
+
|
18
|
+
it "sets polling_interval to 60" do
|
19
|
+
@config.polling_interval.must_equal 60
|
20
|
+
end
|
21
|
+
|
22
|
+
it "sets redis_ns_ryespy to ryespy:" do
|
23
|
+
@config.redis_ns_ryespy.must_equal 'ryespy:'
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
describe "configure block" do
|
28
|
+
before do
|
29
|
+
Ryespy.configure do |c|
|
30
|
+
c.log_level = 'ERROR'
|
31
|
+
c.listener = 'imap'
|
32
|
+
c.polling_interval = 13
|
33
|
+
c.redis_url = 'redis://127.0.0.1:6379/1'
|
34
|
+
c.redis_ns_ryespy = 'WithMyLittleEye!'
|
35
|
+
end
|
36
|
+
|
37
|
+
@config = Ryespy.config
|
38
|
+
end
|
39
|
+
|
40
|
+
it "configures log_level" do
|
41
|
+
@config.log_level.must_equal 'ERROR'
|
42
|
+
end
|
43
|
+
|
44
|
+
it "configures listener" do
|
45
|
+
@config.listener.must_equal 'imap'
|
46
|
+
end
|
47
|
+
|
48
|
+
it "configures polling_interval" do
|
49
|
+
@config.polling_interval.must_equal 13
|
50
|
+
end
|
51
|
+
|
52
|
+
it "configures redis_url" do
|
53
|
+
@config.redis_url.must_equal 'redis://127.0.0.1:6379/1'
|
54
|
+
end
|
55
|
+
|
56
|
+
it "configures redis_ns_ryespy" do
|
57
|
+
@config.redis_ns_ryespy.must_equal 'WithMyLittleEye!'
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
describe "#to_s" do
|
62
|
+
before do
|
63
|
+
@config = Ryespy::Config.new
|
64
|
+
end
|
65
|
+
|
66
|
+
it "stringifies hash of config" do
|
67
|
+
@config.to_s.must_equal '{:log_level=>"INFO", :listener=>nil, :polling_interval=>60, :redis_url=>nil, :redis_ns_ryespy=>"ryespy:", :notifiers=>{:sidekiq=>[]}}'
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
end
|
metadata
ADDED
@@ -0,0 +1,113 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: ryespy
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.6.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- tiredpixel
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-06-24 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: redis
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 3.0.4
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 3.0.4
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: bundler
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ~>
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.3'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ~>
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.3'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
description: |-
|
56
|
+
Ryespy provides a simple executable for listening to
|
57
|
+
IMAP mailboxes or FTP folders, keeps track of what it's seen using Redis,
|
58
|
+
and notifies Redis in a way in which Resque and Sidekiq can process using
|
59
|
+
workers.
|
60
|
+
email:
|
61
|
+
- tp@tiredpixel.com
|
62
|
+
executables:
|
63
|
+
- ryespy.rb
|
64
|
+
extensions: []
|
65
|
+
extra_rdoc_files: []
|
66
|
+
files:
|
67
|
+
- .gitignore
|
68
|
+
- .ruby-gemset
|
69
|
+
- .ruby-version
|
70
|
+
- Gemfile
|
71
|
+
- LICENSE.txt
|
72
|
+
- README.md
|
73
|
+
- Rakefile
|
74
|
+
- bin/ryespy.rb
|
75
|
+
- lib/ryespy.rb
|
76
|
+
- lib/ryespy/config.rb
|
77
|
+
- lib/ryespy/listeners/ftp.rb
|
78
|
+
- lib/ryespy/listeners/imap.rb
|
79
|
+
- lib/ryespy/notifiers/sidekiq.rb
|
80
|
+
- lib/ryespy/redis_conn.rb
|
81
|
+
- lib/ryespy/version.rb
|
82
|
+
- ryespy.gemspec
|
83
|
+
- test/ryespy.rb
|
84
|
+
- test/ryespy/config.rb
|
85
|
+
- test/ryespy/version.rb
|
86
|
+
homepage: https://github.com/tiredpixel/ryespy
|
87
|
+
licenses:
|
88
|
+
- MIT
|
89
|
+
metadata: {}
|
90
|
+
post_install_message:
|
91
|
+
rdoc_options: []
|
92
|
+
require_paths:
|
93
|
+
- lib
|
94
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
95
|
+
requirements:
|
96
|
+
- - '>='
|
97
|
+
- !ruby/object:Gem::Version
|
98
|
+
version: '0'
|
99
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - '>='
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
requirements: []
|
105
|
+
rubyforge_project:
|
106
|
+
rubygems_version: 2.0.3
|
107
|
+
signing_key:
|
108
|
+
specification_version: 4
|
109
|
+
summary: Ryespy listens to IMAP and FTP and queues in Redis (Sidekiq/Resque).
|
110
|
+
test_files:
|
111
|
+
- test/ryespy.rb
|
112
|
+
- test/ryespy/config.rb
|
113
|
+
- test/ryespy/version.rb
|