rest-ftp-daemon 0.250.5 → 0.300.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +19 -14
- data/README.md +12 -3
- data/bin/rest-ftp-daemon +102 -96
- data/config.ru +5 -5
- data/defaults.yml +61 -0
- data/lib/rest-ftp-daemon.rb +10 -4
- data/lib/rest-ftp-daemon/api/config.rb +3 -2
- data/lib/rest-ftp-daemon/api/dashboard.rb +1 -4
- data/lib/rest-ftp-daemon/api/debug.rb +30 -17
- data/lib/rest-ftp-daemon/api/job_presenter.rb +0 -2
- data/lib/rest-ftp-daemon/api/jobs.rb +4 -3
- data/lib/rest-ftp-daemon/api/root.rb +7 -10
- data/lib/rest-ftp-daemon/api/status.rb +7 -13
- data/lib/rest-ftp-daemon/constants.rb +27 -45
- data/lib/rest-ftp-daemon/counters.rb +0 -4
- data/lib/rest-ftp-daemon/helpers.rb +3 -18
- data/lib/rest-ftp-daemon/job.rb +16 -21
- data/lib/rest-ftp-daemon/job_queue.rb +21 -14
- data/lib/rest-ftp-daemon/launcher.rb +26 -0
- data/lib/rest-ftp-daemon/logger_pool.rb +9 -19
- data/lib/rest-ftp-daemon/metrics.rb +41 -0
- data/lib/rest-ftp-daemon/notification.rb +7 -10
- data/lib/rest-ftp-daemon/remote.rb +4 -4
- data/lib/rest-ftp-daemon/remote_ftp.rb +10 -10
- data/lib/rest-ftp-daemon/remote_sftp.rb +13 -24
- data/lib/rest-ftp-daemon/views/dashboard.haml +2 -2
- data/lib/rest-ftp-daemon/views/dashboard_footer.haml +2 -2
- data/lib/rest-ftp-daemon/views/dashboard_header.haml +2 -2
- data/lib/rest-ftp-daemon/views/dashboard_workers.haml +2 -2
- data/lib/rest-ftp-daemon/worker.rb +43 -12
- data/lib/rest-ftp-daemon/worker_conchita.rb +15 -28
- data/lib/rest-ftp-daemon/worker_job.rb +30 -21
- data/lib/rest-ftp-daemon/worker_pool.rb +59 -50
- data/lib/rest-ftp-daemon/worker_reporter.rb +70 -0
- data/lib/shared/conf.rb +195 -0
- data/lib/shared/logger_formatter.rb +31 -0
- data/lib/shared/logger_helper.rb +78 -0
- data/rest-ftp-daemon.gemspec +23 -22
- data/{rest-ftp-daemon.yml.sample → rest-ftp-daemon.sample.yml} +10 -7
- data/spec/spec_helper.rb +1 -1
- metadata +30 -12
- data/lib/rest-ftp-daemon/logger.rb +0 -57
- data/lib/rest-ftp-daemon/logger_helper.rb +0 -36
- data/lib/rest-ftp-daemon/settings.rb +0 -57
@@ -0,0 +1,70 @@
|
|
1
|
+
module RestFtpDaemon
|
2
|
+
|
3
|
+
# Worker used to clean up the queue deleting expired jobs
|
4
|
+
class ReporterWorker < Worker
|
5
|
+
|
6
|
+
def initialize wid, pool = nil
|
7
|
+
# Call dady and load my conf
|
8
|
+
super
|
9
|
+
|
10
|
+
# Start main loop
|
11
|
+
log_info "#{self.class.name} starting", @config
|
12
|
+
start
|
13
|
+
end
|
14
|
+
|
15
|
+
protected
|
16
|
+
|
17
|
+
# def log_prefix
|
18
|
+
# [
|
19
|
+
# Thread.current.thread_variable_get(:wid),
|
20
|
+
# nil,
|
21
|
+
# nil
|
22
|
+
# ]
|
23
|
+
# end
|
24
|
+
|
25
|
+
def work
|
26
|
+
# Announce we are working
|
27
|
+
worker_status WORKER_STATUS_REPORTING
|
28
|
+
|
29
|
+
# Report metrics
|
30
|
+
do_metrics
|
31
|
+
|
32
|
+
rescue StandardError => e
|
33
|
+
log_error "EXCEPTION: #{e.inspect}"
|
34
|
+
sleep 1
|
35
|
+
else
|
36
|
+
wait_according_to_config
|
37
|
+
end
|
38
|
+
|
39
|
+
def maxage status
|
40
|
+
@config["clean_#{status}"] || 0
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def do_metrics
|
46
|
+
# Get common metrics
|
47
|
+
log_info "collecting metrics"
|
48
|
+
metrics = Metrics.sample
|
49
|
+
|
50
|
+
# Dump metrics to logs
|
51
|
+
log_debug "collected metrics", metrics
|
52
|
+
|
53
|
+
# Transpose metrics to NewRelic metrics
|
54
|
+
report_newrelic(metrics) if Conf.newrelic_enabled?
|
55
|
+
end
|
56
|
+
|
57
|
+
def report_newrelic metrics
|
58
|
+
metrics_newrelic = {}
|
59
|
+
metrics.each do |group, pairs|
|
60
|
+
pairs.each do |key, value|
|
61
|
+
name = "rftpd/#{group}/#{key}"
|
62
|
+
::NewRelic::Agent.record_metric(name, value)
|
63
|
+
metrics_newrelic[name] = value
|
64
|
+
end
|
65
|
+
end
|
66
|
+
log_debug "reported [#{metrics.size}] metrics to NewRelic", metrics_newrelic
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
end
|
data/lib/shared/conf.rb
ADDED
@@ -0,0 +1,195 @@
|
|
1
|
+
# FIXME: files named with hyphens will not be found by Chamber for now
|
2
|
+
require "chamber"
|
3
|
+
|
4
|
+
module Shared
|
5
|
+
class ConfigMissingParameter < StandardError; end
|
6
|
+
class ConfigOtherError < StandardError; end
|
7
|
+
class ConfigParseError < StandardError; end
|
8
|
+
class ConfigMultipleGemspec < StandardError; end
|
9
|
+
class ConfigMissingGemspec < StandardError; end
|
10
|
+
|
11
|
+
class Conf
|
12
|
+
extend Chamber
|
13
|
+
|
14
|
+
class << self
|
15
|
+
attr_accessor :app_env
|
16
|
+
attr_reader :app_root
|
17
|
+
attr_reader :app_libs
|
18
|
+
attr_reader :app_name
|
19
|
+
attr_reader :app_ver
|
20
|
+
attr_reader :app_started
|
21
|
+
attr_reader :app_spec
|
22
|
+
attr_reader :files
|
23
|
+
attr_reader :host
|
24
|
+
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.init app_root
|
28
|
+
# Permanent flags
|
29
|
+
@initialized = true
|
30
|
+
@app_started = Time.now
|
31
|
+
|
32
|
+
# Default values
|
33
|
+
@files ||= []
|
34
|
+
@app_name ||= "app_name"
|
35
|
+
@app_env ||= "production"
|
36
|
+
@host ||= `hostname`.to_s.chomp.split(".").first
|
37
|
+
|
38
|
+
# Store and clean app_root
|
39
|
+
@app_root = File.expand_path(app_root)
|
40
|
+
|
41
|
+
# Try to find any gemspec file
|
42
|
+
matches = Dir["#{@app_root}/*.gemspec"]
|
43
|
+
fail ConfigMissingGemspec, "gemspec file not found: #{gemspec_path}" if matches.size < 1
|
44
|
+
fail ConfigMultipleGemspec, "gemspec file not found: #{gemspec_path}" if matches.size > 1
|
45
|
+
|
46
|
+
# Load Gemspec (just the only match)
|
47
|
+
@spec = Gem::Specification::load(matches.first)
|
48
|
+
@app_name = @spec.name
|
49
|
+
@app_ver = @spec.version
|
50
|
+
fail ConfigMissingParameter, "gemspec: missing name" unless @app_name
|
51
|
+
fail ConfigMissingParameter, "gemspec: missing version" unless @app_ver
|
52
|
+
|
53
|
+
# Now we know app_name, initalize app_libs
|
54
|
+
@app_libs = File.expand_path( @app_root + "/lib/#{@app_name}/" )
|
55
|
+
|
56
|
+
# Add other config files
|
57
|
+
add_default_config
|
58
|
+
add_etc_config
|
59
|
+
|
60
|
+
# Return something
|
61
|
+
return @app_name
|
62
|
+
end
|
63
|
+
|
64
|
+
def self.prepare args = {}
|
65
|
+
ensure_init
|
66
|
+
|
67
|
+
# Add extra config file
|
68
|
+
add_extra_config args[:config]
|
69
|
+
|
70
|
+
# Load configuration files
|
71
|
+
load_files
|
72
|
+
|
73
|
+
# Set Rack env
|
74
|
+
ENV["RACK_ENV"] = @app_env.to_s
|
75
|
+
|
76
|
+
# Init New Relic
|
77
|
+
prepare_newrelic self[:newrelic], self.at(:logs, :newrelic)
|
78
|
+
|
79
|
+
# Try to access any key to force parsing of the files
|
80
|
+
self[:dummy]
|
81
|
+
|
82
|
+
rescue Psych::SyntaxError => e
|
83
|
+
fail ConfigParseError, e.message
|
84
|
+
rescue StandardError => e
|
85
|
+
fail ConfigOtherError, "#{e.message} \n #{e.backtrace.to_yaml}"
|
86
|
+
end
|
87
|
+
|
88
|
+
# Reload files
|
89
|
+
def self.reload!
|
90
|
+
ensure_init
|
91
|
+
load_files
|
92
|
+
end
|
93
|
+
|
94
|
+
def self.dump
|
95
|
+
ensure_init
|
96
|
+
to_hash.to_yaml(indent: 4, useheader: true, useversion: false )
|
97
|
+
end
|
98
|
+
|
99
|
+
# Direct access to any depth
|
100
|
+
def self.at *path
|
101
|
+
ensure_init
|
102
|
+
path.reduce(Conf) { |m, key| m && m[key.to_s] }
|
103
|
+
end
|
104
|
+
|
105
|
+
def self.newrelic_enabled?
|
106
|
+
ensure_init
|
107
|
+
!!self[:newrelic]
|
108
|
+
end
|
109
|
+
|
110
|
+
# Defaults generators
|
111
|
+
def self.gen_pidfile
|
112
|
+
ensure_init
|
113
|
+
"/tmp/#{@app_name}-#{@host}-#{self[:port]}.pid"
|
114
|
+
end
|
115
|
+
def self.gen_config_etc
|
116
|
+
ensure_init
|
117
|
+
"/etc/#{@app_name}.yml"
|
118
|
+
end
|
119
|
+
def self.gen_config_sample
|
120
|
+
ensure_init
|
121
|
+
"#{@app_root}/#{@app_name}.sample.yml"
|
122
|
+
end
|
123
|
+
def self.gen_config_message
|
124
|
+
config_etc = gen_config_etc
|
125
|
+
config_sample = gen_config_sample
|
126
|
+
return "
|
127
|
+
A default configuration is available here: #{config_sample}.
|
128
|
+
You should copy it to the default location: #{config_etc}.
|
129
|
+
sudo cp #{config_sample} #{config_etc}
|
130
|
+
"
|
131
|
+
end
|
132
|
+
|
133
|
+
protected
|
134
|
+
|
135
|
+
def self.load_files
|
136
|
+
load files: @files, namespaces: { environment: @app_env }
|
137
|
+
end
|
138
|
+
|
139
|
+
def self.add_default_config
|
140
|
+
@files << "#{@app_root}/defaults.yml" if @app_root
|
141
|
+
end
|
142
|
+
|
143
|
+
def self.add_etc_config
|
144
|
+
@files << File.expand_path("/etc/#{@app_name}.yml") if @app_name
|
145
|
+
end
|
146
|
+
|
147
|
+
def self.add_extra_config path
|
148
|
+
@files << File.expand_path(path) if path
|
149
|
+
end
|
150
|
+
|
151
|
+
def self.prepare_newrelic section, logfile
|
152
|
+
# Disable NewRelic if no config present
|
153
|
+
unless section.is_a?(Hash) && section[:licence]
|
154
|
+
ENV["NEWRELIC_AGENT_ENABLED"] = "false"
|
155
|
+
return
|
156
|
+
end
|
157
|
+
|
158
|
+
# Enable GC profiler
|
159
|
+
GC::Profiler.enable
|
160
|
+
|
161
|
+
# Enable module
|
162
|
+
ENV["NEWRELIC_AGENT_ENABLED"] = "true"
|
163
|
+
ENV["NEW_RELIC_MONITOR_MODE"] = "true"
|
164
|
+
|
165
|
+
# License
|
166
|
+
ENV["NEW_RELIC_LICENSE_KEY"] = section[:licence].to_s
|
167
|
+
|
168
|
+
# Build NewRelic app_name if not provided as-is
|
169
|
+
if section[:app_name]
|
170
|
+
ENV["NEW_RELIC_APP_NAME"] = section[:app_name].to_s
|
171
|
+
else
|
172
|
+
stack = []
|
173
|
+
stack << (section[:prefix] || @app_name)
|
174
|
+
stack << section[:platform] if section[:platform]
|
175
|
+
stack << @app_env
|
176
|
+
text = stack.join('-')
|
177
|
+
ENV["NEW_RELIC_APP_NAME"] = "#{text}-#{host};#{text}"
|
178
|
+
end
|
179
|
+
|
180
|
+
# Logfile
|
181
|
+
ENV["NEW_RELIC_LOG"] = logfile.to_s if logfile
|
182
|
+
end
|
183
|
+
|
184
|
+
private
|
185
|
+
|
186
|
+
def self.ensure_init
|
187
|
+
# Skip is already done
|
188
|
+
return if @initialized
|
189
|
+
|
190
|
+
# Go through init if not already done
|
191
|
+
self.init
|
192
|
+
end
|
193
|
+
|
194
|
+
end
|
195
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module Shared
|
2
|
+
class LoggerFormatter
|
3
|
+
|
4
|
+
def self.call severity, datetime, progname, payload
|
5
|
+
# Build common values
|
6
|
+
timestamp = datetime.strftime(LOG_HEADER_TIME)
|
7
|
+
|
8
|
+
# Build header
|
9
|
+
header = sprintf LOG_HEADER_FORMAT,
|
10
|
+
timestamp,
|
11
|
+
Process.pid,
|
12
|
+
severity,
|
13
|
+
progname
|
14
|
+
|
15
|
+
# If we have a bunch of lines, prefix them and send them together
|
16
|
+
return payload.map do |line|
|
17
|
+
"#{header}#{trimmed(line)}\n"
|
18
|
+
end.join if payload.is_a?(Array)
|
19
|
+
|
20
|
+
# Otherwise, just prefix the only line
|
21
|
+
return "#{header}#{trimmed(payload)}\n"
|
22
|
+
end
|
23
|
+
|
24
|
+
protected
|
25
|
+
|
26
|
+
def self.trimmed line
|
27
|
+
line.to_s.rstrip[0..LOG_MESSAGE_TRIM].force_encoding(Encoding::UTF_8)
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,78 @@
|
|
1
|
+
require "logger"
|
2
|
+
|
3
|
+
module Shared
|
4
|
+
module LoggerHelper
|
5
|
+
|
6
|
+
protected
|
7
|
+
|
8
|
+
def log_info message, details = nil
|
9
|
+
build_messages Logger::INFO, message, details
|
10
|
+
end
|
11
|
+
|
12
|
+
def log_error message, details = nil
|
13
|
+
build_messages Logger::ERROR, message, details
|
14
|
+
end
|
15
|
+
|
16
|
+
def log_debug message, details = nil
|
17
|
+
build_messages Logger::DEBUG, message, details
|
18
|
+
end
|
19
|
+
|
20
|
+
alias info log_info
|
21
|
+
alias error log_error
|
22
|
+
alias debug log_debug
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
# Builds prefix if LOG_PREFIX_FORMAT defined and caller has log_prefix method to provide values
|
27
|
+
def build_prefix
|
28
|
+
# Skip if no values from user class
|
29
|
+
return unless respond_to?(:log_prefix, true)
|
30
|
+
values = log_prefix
|
31
|
+
|
32
|
+
# Skip if no format defined
|
33
|
+
return unless defined?('LOG_PREFIX_FORMAT')
|
34
|
+
return unless LOG_PREFIX_FORMAT.is_a? String
|
35
|
+
|
36
|
+
# Build prefix string
|
37
|
+
LOG_PREFIX_FORMAT % values.map(&:to_s)
|
38
|
+
end
|
39
|
+
|
40
|
+
def build_messages severity, message, details = nil
|
41
|
+
messages = []
|
42
|
+
# messages << "/---------------------------------------"
|
43
|
+
# messages << "severity: #{severity}"
|
44
|
+
# messages << "message: #{message.class}"
|
45
|
+
# messages << "details: #{details.class} #{details.inspect}"
|
46
|
+
# messages << "ARRAY(#{details.count})" if details.is_a? Array
|
47
|
+
# messages << "HASH(#{details.count})" if details.is_a? Hash
|
48
|
+
|
49
|
+
prefix = build_prefix
|
50
|
+
|
51
|
+
# Add main message
|
52
|
+
messages << sprintf(LOG_MESSAGE_TEXT, prefix, message) if message
|
53
|
+
|
54
|
+
# Add details from array
|
55
|
+
details.each do |line|
|
56
|
+
messages << sprintf(LOG_MESSAGE_ARRAY, prefix, line)
|
57
|
+
end if details.is_a? Array
|
58
|
+
|
59
|
+
# Add details from hash
|
60
|
+
details.each do |key, value|
|
61
|
+
messages << sprintf(LOG_MESSAGE_HASH, prefix, key, value)
|
62
|
+
end if details.is_a? Hash
|
63
|
+
|
64
|
+
# Return all that stuff
|
65
|
+
# messages << "\\---------------------------------------"
|
66
|
+
logger.add severity, messages
|
67
|
+
end
|
68
|
+
|
69
|
+
# def debug_lines lines, prefix = ''
|
70
|
+
# if lines.is_a? Array
|
71
|
+
# logger.debug lines.map{ |line| sprintf(LOG_MESSAGE_ARRAY, prefix, line) }
|
72
|
+
# elsif lines.is_a? Hash
|
73
|
+
# logger.debug lines.map{ |key, value| sprintf(LOG_MESSAGE_HASH, prefix, key, value) }
|
74
|
+
# end
|
75
|
+
# end
|
76
|
+
|
77
|
+
end
|
78
|
+
end
|
data/rest-ftp-daemon.gemspec
CHANGED
@@ -1,40 +1,41 @@
|
|
1
1
|
# coding: utf-8
|
2
|
+
Gem::Specification.new do |spec|
|
2
3
|
|
3
|
-
#
|
4
|
-
|
5
|
-
|
4
|
+
# Project version
|
5
|
+
spec.version = "0.300.1"
|
6
6
|
|
7
|
-
#
|
8
|
-
|
9
|
-
spec.
|
10
|
-
spec.
|
11
|
-
spec.
|
12
|
-
spec.
|
13
|
-
spec.
|
14
|
-
spec.
|
15
|
-
spec.
|
16
|
-
spec.licenses = ["MIT"]
|
7
|
+
# Project description
|
8
|
+
spec.name = "rest-ftp-daemon"
|
9
|
+
spec.authors = ["Bruno MEDICI"]
|
10
|
+
spec.email = "rest-ftp-daemon@bmconseil.com"
|
11
|
+
spec.description = "This is a pretty simple FTP client daemon, controlled through a RESTful API"
|
12
|
+
spec.summary = "RESTful FTP client daemon"
|
13
|
+
spec.homepage = "http://github.com/bmedici/rest-ftp-daemon"
|
14
|
+
spec.licenses = ["MIT"]
|
15
|
+
spec.date = Time.now.strftime("%Y-%m-%d")
|
17
16
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
spec.
|
22
|
-
spec.
|
23
|
-
spec.version = APP_VER
|
17
|
+
# List files and executables
|
18
|
+
spec.files = `git ls-files -z`.split("\x0").reject{ |f| f == "dashboard.png"}
|
19
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
20
|
+
spec.require_paths = ["lib"]
|
21
|
+
spec.required_ruby_version = ">= 2.2"
|
24
22
|
|
25
|
-
spec.required_ruby_version = ">= 2.2"
|
26
23
|
|
24
|
+
# Development dependencies
|
27
25
|
spec.add_development_dependency "bundler", "~> 1.6"
|
28
26
|
spec.add_development_dependency "rake"
|
29
27
|
spec.add_development_dependency "rspec"
|
30
|
-
spec.add_development_dependency "http"
|
28
|
+
spec.add_development_dependency "http"
|
29
|
+
#spec.add_development_dependency "http", "~> 0.8"
|
31
30
|
spec.add_development_dependency "rubocop", "~> 0.32.0"
|
32
31
|
spec.add_development_dependency "pry"
|
33
32
|
|
34
|
-
|
33
|
+
# Runtime dependencies
|
34
|
+
spec.add_runtime_dependency "thin", "~> 1"
|
35
35
|
spec.add_runtime_dependency "grape"
|
36
36
|
spec.add_runtime_dependency "grape-entity"
|
37
37
|
spec.add_runtime_dependency "settingslogic"
|
38
|
+
spec.add_runtime_dependency "chamber"
|
38
39
|
spec.add_runtime_dependency "haml"
|
39
40
|
spec.add_runtime_dependency "json"
|
40
41
|
spec.add_runtime_dependency "net-sftp"
|
@@ -3,7 +3,8 @@ defaults: &defaults
|
|
3
3
|
port: 3000
|
4
4
|
user: rftpd
|
5
5
|
group: rftpd
|
6
|
-
#host: "myhost"
|
6
|
+
# host: "myhost"
|
7
|
+
# pidfile: "/tmp/rftpd.pid"
|
7
8
|
|
8
9
|
pools:
|
9
10
|
default: 2
|
@@ -39,15 +40,17 @@ defaults: &defaults
|
|
39
40
|
# clean_queued: 86400
|
40
41
|
|
41
42
|
newrelic:
|
42
|
-
licence: "
|
43
|
+
licence: ""
|
44
|
+
# prefix: "rftpd" # app prefix
|
43
45
|
platform: "bigbusiness" # app platform name
|
44
|
-
app_name: "rftpd-bigbusiness-dev" # nickname used for naming app
|
46
|
+
# app_name: "rftpd-bigbusiness-dev" # nickname used for naming app
|
45
47
|
|
46
48
|
debug:
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
49
|
+
ftp: false
|
50
|
+
sftp: false
|
51
|
+
conchita: false
|
52
|
+
reporter: fakse
|
53
|
+
allow_reload: false
|
51
54
|
|
52
55
|
logs:
|
53
56
|
thin: "/var/log/rftpd-environment-thin.log"
|