rest-ftp-daemon 0.250.5 → 0.300.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +19 -14
  3. data/README.md +12 -3
  4. data/bin/rest-ftp-daemon +102 -96
  5. data/config.ru +5 -5
  6. data/defaults.yml +61 -0
  7. data/lib/rest-ftp-daemon.rb +10 -4
  8. data/lib/rest-ftp-daemon/api/config.rb +3 -2
  9. data/lib/rest-ftp-daemon/api/dashboard.rb +1 -4
  10. data/lib/rest-ftp-daemon/api/debug.rb +30 -17
  11. data/lib/rest-ftp-daemon/api/job_presenter.rb +0 -2
  12. data/lib/rest-ftp-daemon/api/jobs.rb +4 -3
  13. data/lib/rest-ftp-daemon/api/root.rb +7 -10
  14. data/lib/rest-ftp-daemon/api/status.rb +7 -13
  15. data/lib/rest-ftp-daemon/constants.rb +27 -45
  16. data/lib/rest-ftp-daemon/counters.rb +0 -4
  17. data/lib/rest-ftp-daemon/helpers.rb +3 -18
  18. data/lib/rest-ftp-daemon/job.rb +16 -21
  19. data/lib/rest-ftp-daemon/job_queue.rb +21 -14
  20. data/lib/rest-ftp-daemon/launcher.rb +26 -0
  21. data/lib/rest-ftp-daemon/logger_pool.rb +9 -19
  22. data/lib/rest-ftp-daemon/metrics.rb +41 -0
  23. data/lib/rest-ftp-daemon/notification.rb +7 -10
  24. data/lib/rest-ftp-daemon/remote.rb +4 -4
  25. data/lib/rest-ftp-daemon/remote_ftp.rb +10 -10
  26. data/lib/rest-ftp-daemon/remote_sftp.rb +13 -24
  27. data/lib/rest-ftp-daemon/views/dashboard.haml +2 -2
  28. data/lib/rest-ftp-daemon/views/dashboard_footer.haml +2 -2
  29. data/lib/rest-ftp-daemon/views/dashboard_header.haml +2 -2
  30. data/lib/rest-ftp-daemon/views/dashboard_workers.haml +2 -2
  31. data/lib/rest-ftp-daemon/worker.rb +43 -12
  32. data/lib/rest-ftp-daemon/worker_conchita.rb +15 -28
  33. data/lib/rest-ftp-daemon/worker_job.rb +30 -21
  34. data/lib/rest-ftp-daemon/worker_pool.rb +59 -50
  35. data/lib/rest-ftp-daemon/worker_reporter.rb +70 -0
  36. data/lib/shared/conf.rb +195 -0
  37. data/lib/shared/logger_formatter.rb +31 -0
  38. data/lib/shared/logger_helper.rb +78 -0
  39. data/rest-ftp-daemon.gemspec +23 -22
  40. data/{rest-ftp-daemon.yml.sample → rest-ftp-daemon.sample.yml} +10 -7
  41. data/spec/spec_helper.rb +1 -1
  42. metadata +30 -12
  43. data/lib/rest-ftp-daemon/logger.rb +0 -57
  44. data/lib/rest-ftp-daemon/logger_helper.rb +0 -36
  45. data/lib/rest-ftp-daemon/settings.rb +0 -57
@@ -0,0 +1,70 @@
1
+ module RestFtpDaemon
2
+
3
+ # Worker used to clean up the queue deleting expired jobs
4
+ class ReporterWorker < Worker
5
+
6
+ def initialize wid, pool = nil
7
+ # Call dady and load my conf
8
+ super
9
+
10
+ # Start main loop
11
+ log_info "#{self.class.name} starting", @config
12
+ start
13
+ end
14
+
15
+ protected
16
+
17
+ # def log_prefix
18
+ # [
19
+ # Thread.current.thread_variable_get(:wid),
20
+ # nil,
21
+ # nil
22
+ # ]
23
+ # end
24
+
25
+ def work
26
+ # Announce we are working
27
+ worker_status WORKER_STATUS_REPORTING
28
+
29
+ # Report metrics
30
+ do_metrics
31
+
32
+ rescue StandardError => e
33
+ log_error "EXCEPTION: #{e.inspect}"
34
+ sleep 1
35
+ else
36
+ wait_according_to_config
37
+ end
38
+
39
+ def maxage status
40
+ @config["clean_#{status}"] || 0
41
+ end
42
+
43
+ private
44
+
45
+ def do_metrics
46
+ # Get common metrics
47
+ log_info "collecting metrics"
48
+ metrics = Metrics.sample
49
+
50
+ # Dump metrics to logs
51
+ log_debug "collected metrics", metrics
52
+
53
+ # Transpose metrics to NewRelic metrics
54
+ report_newrelic(metrics) if Conf.newrelic_enabled?
55
+ end
56
+
57
+ def report_newrelic metrics
58
+ metrics_newrelic = {}
59
+ metrics.each do |group, pairs|
60
+ pairs.each do |key, value|
61
+ name = "rftpd/#{group}/#{key}"
62
+ ::NewRelic::Agent.record_metric(name, value)
63
+ metrics_newrelic[name] = value
64
+ end
65
+ end
66
+ log_debug "reported [#{metrics.size}] metrics to NewRelic", metrics_newrelic
67
+ end
68
+
69
+ end
70
+ end
@@ -0,0 +1,195 @@
1
+ # FIXME: files named with hyphens will not be found by Chamber for now
2
+ require "chamber"
3
+
4
+ module Shared
5
+ class ConfigMissingParameter < StandardError; end
6
+ class ConfigOtherError < StandardError; end
7
+ class ConfigParseError < StandardError; end
8
+ class ConfigMultipleGemspec < StandardError; end
9
+ class ConfigMissingGemspec < StandardError; end
10
+
11
+ class Conf
12
+ extend Chamber
13
+
14
+ class << self
15
+ attr_accessor :app_env
16
+ attr_reader :app_root
17
+ attr_reader :app_libs
18
+ attr_reader :app_name
19
+ attr_reader :app_ver
20
+ attr_reader :app_started
21
+ attr_reader :app_spec
22
+ attr_reader :files
23
+ attr_reader :host
24
+
25
+ end
26
+
27
+ def self.init app_root
28
+ # Permanent flags
29
+ @initialized = true
30
+ @app_started = Time.now
31
+
32
+ # Default values
33
+ @files ||= []
34
+ @app_name ||= "app_name"
35
+ @app_env ||= "production"
36
+ @host ||= `hostname`.to_s.chomp.split(".").first
37
+
38
+ # Store and clean app_root
39
+ @app_root = File.expand_path(app_root)
40
+
41
+ # Try to find any gemspec file
42
+ matches = Dir["#{@app_root}/*.gemspec"]
43
+ fail ConfigMissingGemspec, "gemspec file not found: #{gemspec_path}" if matches.size < 1
44
+ fail ConfigMultipleGemspec, "gemspec file not found: #{gemspec_path}" if matches.size > 1
45
+
46
+ # Load Gemspec (just the only match)
47
+ @spec = Gem::Specification::load(matches.first)
48
+ @app_name = @spec.name
49
+ @app_ver = @spec.version
50
+ fail ConfigMissingParameter, "gemspec: missing name" unless @app_name
51
+ fail ConfigMissingParameter, "gemspec: missing version" unless @app_ver
52
+
53
+ # Now we know app_name, initalize app_libs
54
+ @app_libs = File.expand_path( @app_root + "/lib/#{@app_name}/" )
55
+
56
+ # Add other config files
57
+ add_default_config
58
+ add_etc_config
59
+
60
+ # Return something
61
+ return @app_name
62
+ end
63
+
64
+ def self.prepare args = {}
65
+ ensure_init
66
+
67
+ # Add extra config file
68
+ add_extra_config args[:config]
69
+
70
+ # Load configuration files
71
+ load_files
72
+
73
+ # Set Rack env
74
+ ENV["RACK_ENV"] = @app_env.to_s
75
+
76
+ # Init New Relic
77
+ prepare_newrelic self[:newrelic], self.at(:logs, :newrelic)
78
+
79
+ # Try to access any key to force parsing of the files
80
+ self[:dummy]
81
+
82
+ rescue Psych::SyntaxError => e
83
+ fail ConfigParseError, e.message
84
+ rescue StandardError => e
85
+ fail ConfigOtherError, "#{e.message} \n #{e.backtrace.to_yaml}"
86
+ end
87
+
88
+ # Reload files
89
+ def self.reload!
90
+ ensure_init
91
+ load_files
92
+ end
93
+
94
+ def self.dump
95
+ ensure_init
96
+ to_hash.to_yaml(indent: 4, useheader: true, useversion: false )
97
+ end
98
+
99
+ # Direct access to any depth
100
+ def self.at *path
101
+ ensure_init
102
+ path.reduce(Conf) { |m, key| m && m[key.to_s] }
103
+ end
104
+
105
+ def self.newrelic_enabled?
106
+ ensure_init
107
+ !!self[:newrelic]
108
+ end
109
+
110
+ # Defaults generators
111
+ def self.gen_pidfile
112
+ ensure_init
113
+ "/tmp/#{@app_name}-#{@host}-#{self[:port]}.pid"
114
+ end
115
+ def self.gen_config_etc
116
+ ensure_init
117
+ "/etc/#{@app_name}.yml"
118
+ end
119
+ def self.gen_config_sample
120
+ ensure_init
121
+ "#{@app_root}/#{@app_name}.sample.yml"
122
+ end
123
+ def self.gen_config_message
124
+ config_etc = gen_config_etc
125
+ config_sample = gen_config_sample
126
+ return "
127
+ A default configuration is available here: #{config_sample}.
128
+ You should copy it to the default location: #{config_etc}.
129
+ sudo cp #{config_sample} #{config_etc}
130
+ "
131
+ end
132
+
133
+ protected
134
+
135
+ def self.load_files
136
+ load files: @files, namespaces: { environment: @app_env }
137
+ end
138
+
139
+ def self.add_default_config
140
+ @files << "#{@app_root}/defaults.yml" if @app_root
141
+ end
142
+
143
+ def self.add_etc_config
144
+ @files << File.expand_path("/etc/#{@app_name}.yml") if @app_name
145
+ end
146
+
147
+ def self.add_extra_config path
148
+ @files << File.expand_path(path) if path
149
+ end
150
+
151
+ def self.prepare_newrelic section, logfile
152
+ # Disable NewRelic if no config present
153
+ unless section.is_a?(Hash) && section[:licence]
154
+ ENV["NEWRELIC_AGENT_ENABLED"] = "false"
155
+ return
156
+ end
157
+
158
+ # Enable GC profiler
159
+ GC::Profiler.enable
160
+
161
+ # Enable module
162
+ ENV["NEWRELIC_AGENT_ENABLED"] = "true"
163
+ ENV["NEW_RELIC_MONITOR_MODE"] = "true"
164
+
165
+ # License
166
+ ENV["NEW_RELIC_LICENSE_KEY"] = section[:licence].to_s
167
+
168
+ # Build NewRelic app_name if not provided as-is
169
+ if section[:app_name]
170
+ ENV["NEW_RELIC_APP_NAME"] = section[:app_name].to_s
171
+ else
172
+ stack = []
173
+ stack << (section[:prefix] || @app_name)
174
+ stack << section[:platform] if section[:platform]
175
+ stack << @app_env
176
+ text = stack.join('-')
177
+ ENV["NEW_RELIC_APP_NAME"] = "#{text}-#{host};#{text}"
178
+ end
179
+
180
+ # Logfile
181
+ ENV["NEW_RELIC_LOG"] = logfile.to_s if logfile
182
+ end
183
+
184
+ private
185
+
186
+ def self.ensure_init
187
+ # Skip is already done
188
+ return if @initialized
189
+
190
+ # Go through init if not already done
191
+ self.init
192
+ end
193
+
194
+ end
195
+ end
@@ -0,0 +1,31 @@
1
+ module Shared
2
+ class LoggerFormatter
3
+
4
+ def self.call severity, datetime, progname, payload
5
+ # Build common values
6
+ timestamp = datetime.strftime(LOG_HEADER_TIME)
7
+
8
+ # Build header
9
+ header = sprintf LOG_HEADER_FORMAT,
10
+ timestamp,
11
+ Process.pid,
12
+ severity,
13
+ progname
14
+
15
+ # If we have a bunch of lines, prefix them and send them together
16
+ return payload.map do |line|
17
+ "#{header}#{trimmed(line)}\n"
18
+ end.join if payload.is_a?(Array)
19
+
20
+ # Otherwise, just prefix the only line
21
+ return "#{header}#{trimmed(payload)}\n"
22
+ end
23
+
24
+ protected
25
+
26
+ def self.trimmed line
27
+ line.to_s.rstrip[0..LOG_MESSAGE_TRIM].force_encoding(Encoding::UTF_8)
28
+ end
29
+
30
+ end
31
+ end
@@ -0,0 +1,78 @@
1
+ require "logger"
2
+
3
+ module Shared
4
+ module LoggerHelper
5
+
6
+ protected
7
+
8
+ def log_info message, details = nil
9
+ build_messages Logger::INFO, message, details
10
+ end
11
+
12
+ def log_error message, details = nil
13
+ build_messages Logger::ERROR, message, details
14
+ end
15
+
16
+ def log_debug message, details = nil
17
+ build_messages Logger::DEBUG, message, details
18
+ end
19
+
20
+ alias info log_info
21
+ alias error log_error
22
+ alias debug log_debug
23
+
24
+ private
25
+
26
+ # Builds prefix if LOG_PREFIX_FORMAT defined and caller has log_prefix method to provide values
27
+ def build_prefix
28
+ # Skip if no values from user class
29
+ return unless respond_to?(:log_prefix, true)
30
+ values = log_prefix
31
+
32
+ # Skip if no format defined
33
+ return unless defined?('LOG_PREFIX_FORMAT')
34
+ return unless LOG_PREFIX_FORMAT.is_a? String
35
+
36
+ # Build prefix string
37
+ LOG_PREFIX_FORMAT % values.map(&:to_s)
38
+ end
39
+
40
+ def build_messages severity, message, details = nil
41
+ messages = []
42
+ # messages << "/---------------------------------------"
43
+ # messages << "severity: #{severity}"
44
+ # messages << "message: #{message.class}"
45
+ # messages << "details: #{details.class} #{details.inspect}"
46
+ # messages << "ARRAY(#{details.count})" if details.is_a? Array
47
+ # messages << "HASH(#{details.count})" if details.is_a? Hash
48
+
49
+ prefix = build_prefix
50
+
51
+ # Add main message
52
+ messages << sprintf(LOG_MESSAGE_TEXT, prefix, message) if message
53
+
54
+ # Add details from array
55
+ details.each do |line|
56
+ messages << sprintf(LOG_MESSAGE_ARRAY, prefix, line)
57
+ end if details.is_a? Array
58
+
59
+ # Add details from hash
60
+ details.each do |key, value|
61
+ messages << sprintf(LOG_MESSAGE_HASH, prefix, key, value)
62
+ end if details.is_a? Hash
63
+
64
+ # Return all that stuff
65
+ # messages << "\\---------------------------------------"
66
+ logger.add severity, messages
67
+ end
68
+
69
+ # def debug_lines lines, prefix = ''
70
+ # if lines.is_a? Array
71
+ # logger.debug lines.map{ |line| sprintf(LOG_MESSAGE_ARRAY, prefix, line) }
72
+ # elsif lines.is_a? Hash
73
+ # logger.debug lines.map{ |key, value| sprintf(LOG_MESSAGE_HASH, prefix, key, value) }
74
+ # end
75
+ # end
76
+
77
+ end
78
+ end
@@ -1,40 +1,41 @@
1
1
  # coding: utf-8
2
+ Gem::Specification.new do |spec|
2
3
 
3
- # Libs
4
- require_relative "lib/rest-ftp-daemon/constants"
5
-
4
+ # Project version
5
+ spec.version = "0.300.1"
6
6
 
7
- # Gemspec
8
- Gem::Specification.new do |spec|
9
- spec.name = APP_NAME
10
- spec.date = Time.now.strftime("%Y-%m-%d")
11
- spec.authors = ["Bruno MEDICI"]
12
- spec.email = "rest-ftp-daemon@bmconseil.com"
13
- spec.description = "This is a pretty simple FTP client daemon, controlled through a RESTful API"
14
- spec.summary = "RESTful FTP client daemon"
15
- spec.homepage = "http://github.com/bmedici/rest-ftp-daemon"
16
- spec.licenses = ["MIT"]
7
+ # Project description
8
+ spec.name = "rest-ftp-daemon"
9
+ spec.authors = ["Bruno MEDICI"]
10
+ spec.email = "rest-ftp-daemon@bmconseil.com"
11
+ spec.description = "This is a pretty simple FTP client daemon, controlled through a RESTful API"
12
+ spec.summary = "RESTful FTP client daemon"
13
+ spec.homepage = "http://github.com/bmedici/rest-ftp-daemon"
14
+ spec.licenses = ["MIT"]
15
+ spec.date = Time.now.strftime("%Y-%m-%d")
17
16
 
18
- spec.files = `git ls-files -z`.split("\x0").reject do |f|
19
- f == "dashboard.png"
20
- end
21
- spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
22
- spec.require_paths = ["lib"]
23
- spec.version = APP_VER
17
+ # List files and executables
18
+ spec.files = `git ls-files -z`.split("\x0").reject{ |f| f == "dashboard.png"}
19
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
20
+ spec.require_paths = ["lib"]
21
+ spec.required_ruby_version = ">= 2.2"
24
22
 
25
- spec.required_ruby_version = ">= 2.2"
26
23
 
24
+ # Development dependencies
27
25
  spec.add_development_dependency "bundler", "~> 1.6"
28
26
  spec.add_development_dependency "rake"
29
27
  spec.add_development_dependency "rspec"
30
- spec.add_development_dependency "http", "~> 0.8"
28
+ spec.add_development_dependency "http"
29
+ #spec.add_development_dependency "http", "~> 0.8"
31
30
  spec.add_development_dependency "rubocop", "~> 0.32.0"
32
31
  spec.add_development_dependency "pry"
33
32
 
34
- spec.add_runtime_dependency "thin", "~> 1.6"
33
+ # Runtime dependencies
34
+ spec.add_runtime_dependency "thin", "~> 1"
35
35
  spec.add_runtime_dependency "grape"
36
36
  spec.add_runtime_dependency "grape-entity"
37
37
  spec.add_runtime_dependency "settingslogic"
38
+ spec.add_runtime_dependency "chamber"
38
39
  spec.add_runtime_dependency "haml"
39
40
  spec.add_runtime_dependency "json"
40
41
  spec.add_runtime_dependency "net-sftp"
@@ -3,7 +3,8 @@ defaults: &defaults
3
3
  port: 3000
4
4
  user: rftpd
5
5
  group: rftpd
6
- #host: "myhost"
6
+ # host: "myhost"
7
+ # pidfile: "/tmp/rftpd.pid"
7
8
 
8
9
  pools:
9
10
  default: 2
@@ -39,15 +40,17 @@ defaults: &defaults
39
40
  # clean_queued: 86400
40
41
 
41
42
  newrelic:
42
- licence: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
43
+ licence: ""
44
+ # prefix: "rftpd" # app prefix
43
45
  platform: "bigbusiness" # app platform name
44
- app_name: "rftpd-bigbusiness-dev" # nickname used for naming app
46
+ # app_name: "rftpd-bigbusiness-dev" # nickname used for naming app
45
47
 
46
48
  debug:
47
- # ftp: false
48
- # sftp: false
49
- # conchita: false
50
- # allow_reload: false
49
+ ftp: false
50
+ sftp: false
51
+ conchita: false
52
+ reporter: fakse
53
+ allow_reload: false
51
54
 
52
55
  logs:
53
56
  thin: "/var/log/rftpd-environment-thin.log"