fig 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,4 @@
1
1
  require 'fig/package_descriptor_parse_error'
2
- require 'fig/statement'
3
2
 
4
3
  module Fig; end
5
4
 
@@ -7,6 +6,8 @@ module Fig; end
7
6
  class Fig::PackageDescriptor
8
7
  include Comparable
9
8
 
9
+ COMPONENT_PATTERN = / \A (?! [.]{1,2} $) [a-zA-Z0-9_.-]+ \z /x
10
+
10
11
  attr_reader :name, :version, :config, :original_string, :description
11
12
 
12
13
  def self.format(
@@ -117,7 +118,7 @@ class Fig::PackageDescriptor
117
118
  def validate_component_format(value, name, options)
118
119
  return if value.nil?
119
120
 
120
- return if value =~ / \A [a-zA-Z0-9_.-]+ \z /x
121
+ return if value =~ COMPONENT_PATTERN
121
122
 
122
123
  raise Fig::PackageDescriptorParseError.new(
123
124
  %Q<Invalid #{name} ("#{value}")#{standard_exception_suffix(options)}>,
@@ -0,0 +1,47 @@
1
+ require 'fig/logging'
2
+ require 'fig/network_error'
3
+
4
+ module Fig; end
5
+
6
+ # File transfers.
7
+ module Fig::Protocol
8
+ def download_list(uri)
9
+ Fig::Logging.fatal "Protocol not supported: #{uri}"
10
+ raise Fig::NetworkError.new "Protocol not supported: #{uri}"
11
+ end
12
+
13
+ # Determine whether we need to update something. Returns nil to indicate
14
+ # "don't know".
15
+ def path_up_to_date?(uri, path)
16
+ return nil # Not implemented
17
+ end
18
+
19
+ # Returns whether the file was not downloaded because the file already
20
+ # exists and is already up-to-date.
21
+ def download(uri, path)
22
+ Fig::Logging.fatal "Protocol not supported: #{uri}"
23
+ raise Fig::NetworkError.new "Protocol not supported: #{uri}"
24
+ end
25
+
26
+ def upload(local_file, uri)
27
+ Fig::Logging.fatal "Protocol not supported: #{uri}"
28
+ raise Fig::NetworkError.new "Protocol not supported: #{uri}"
29
+ end
30
+
31
+ private
32
+
33
+ def strip_paths_for_list(ls_output, packages, path)
34
+ if not ls_output.nil?
35
+ ls_output = ls_output.gsub(path + '/', '').gsub(path, '').split("\n")
36
+ ls_output.each do |line|
37
+ parts =
38
+ line.gsub(/\\/, '/').sub(/^\.\//, '').sub(/:$/, '').chomp().split('/')
39
+ packages << parts.join('/') if parts.size == 2
40
+ end
41
+ end
42
+ end
43
+
44
+ def log_download(uri, path)
45
+ Fig::Logging.debug "Downloading #{uri} to #{path}."
46
+ end
47
+ end
@@ -0,0 +1,64 @@
1
+ require 'cgi'
2
+ require 'fileutils'
3
+ require 'find'
4
+
5
+ require 'fig/file_not_found_error'
6
+ require 'fig/logging'
7
+ require 'fig/protocol'
8
+
9
+ module Fig; end
10
+ module Fig::Protocol; end
11
+
12
+ # File transfers for the local filesystem.
13
+ class Fig::Protocol::File
14
+ include Fig::Protocol
15
+
16
+ def download_list(uri)
17
+ packages = []
18
+ unescaped_path = CGI.unescape uri.path
19
+ return packages if ! ::File.exist?(unescaped_path)
20
+
21
+ ls = ''
22
+ Find.find(unescaped_path) { |file| ls << file.to_s; ls << "\n" }
23
+
24
+ strip_paths_for_list(ls, packages, unescaped_path)
25
+
26
+ return packages
27
+ end
28
+
29
+ # Determine whether we need to update something. Returns nil to indicate
30
+ # "don't know".
31
+ def path_up_to_date?(uri, path)
32
+ begin
33
+ unescaped_path = CGI.unescape uri.path
34
+ if ::File.mtime(unescaped_path) <= ::File.mtime(path)
35
+ return true
36
+ end
37
+
38
+ return false
39
+ rescue Errno::ENOENT => error
40
+ raise Fig::FileNotFoundError.new error.message, uri
41
+ end
42
+ end
43
+
44
+ # Returns whether the file was not downloaded because the file already
45
+ # exists and is already up-to-date.
46
+ def download(uri, path)
47
+ begin
48
+ unescaped_path = CGI.unescape uri.path
49
+ FileUtils.cp(unescaped_path, path)
50
+
51
+ return true
52
+ rescue Errno::ENOENT => error
53
+ raise Fig::FileNotFoundError.new error.message, uri
54
+ end
55
+ end
56
+
57
+ def upload(local_file, uri)
58
+ unescaped_path = CGI.unescape uri.path
59
+ FileUtils.mkdir_p(::File.dirname(unescaped_path))
60
+ FileUtils.cp(local_file, unescaped_path)
61
+
62
+ return
63
+ end
64
+ end
@@ -0,0 +1,162 @@
1
+ require 'net/ftp'
2
+
3
+ require 'fig/file_not_found_error'
4
+ require 'fig/logging'
5
+ require 'fig/network_error'
6
+ require 'fig/protocol'
7
+ require 'fig/protocol/netrc_enabled'
8
+ require 'fig/url'
9
+
10
+ module Fig; end
11
+ module Fig::Protocol; end
12
+
13
+ # File transfers via FTP
14
+ class Fig::Protocol::FTP
15
+ include Fig::Protocol
16
+ include Fig::Protocol::NetRCEnabled
17
+
18
+ def initialize(login)
19
+ @login = login
20
+ end
21
+
22
+ def download_list(uri)
23
+ ftp = Net::FTP.new(uri.host)
24
+ ftp_login(ftp, uri.host)
25
+ ftp.chdir(uri.path)
26
+ dirs = ftp.nlst
27
+ ftp.close
28
+
29
+ download_ftp_list(uri, dirs)
30
+ end
31
+
32
+ # Determine whether we need to update something. Returns nil to indicate
33
+ # "don't know".
34
+ def path_up_to_date?(uri, path)
35
+ begin
36
+ ftp = Net::FTP.new(uri.host)
37
+ ftp_login(ftp, uri.host)
38
+
39
+ if ftp.mtime(uri.path) <= ::File.mtime(path)
40
+ return true
41
+ end
42
+
43
+ return false
44
+ rescue Net::FTPPermError => error
45
+ Fig::Logging.debug error.message
46
+ raise Fig::FileNotFoundError.new error.message, uri
47
+ rescue SocketError => error
48
+ Fig::Logging.debug error.message
49
+ raise Fig::FileNotFoundError.new error.message, uri
50
+ end
51
+ end
52
+
53
+ # Returns whether the file was not downloaded because the file already
54
+ # exists and is already up-to-date.
55
+ def download(uri, path)
56
+ begin
57
+ ftp = Net::FTP.new(uri.host)
58
+ ftp_login(ftp, uri.host)
59
+
60
+ if ::File.exist?(path) && ftp.mtime(uri.path) <= ::File.mtime(path)
61
+ Fig::Logging.debug "#{path} is up to date."
62
+ return false
63
+ else
64
+ log_download(uri, path)
65
+ ftp.getbinaryfile(uri.path, path, 256*1024)
66
+ return true
67
+ end
68
+ rescue Net::FTPPermError => error
69
+ Fig::Logging.debug error.message
70
+ raise Fig::FileNotFoundError.new error.message, uri
71
+ rescue SocketError => error
72
+ Fig::Logging.debug error.message
73
+ raise Fig::FileNotFoundError.new error.message, uri
74
+ rescue Errno::ETIMEDOUT => error
75
+ Fig::Logging.debug error.message
76
+ raise Fig::FileNotFoundError.new error.message, uri
77
+ end
78
+ end
79
+
80
+ def upload(local_file, uri)
81
+ ftp_uri = Fig::URL.parse(ENV['FIG_REMOTE_URL'])
82
+ ftp_root_path = ftp_uri.path
83
+ ftp_root_dirs = ftp_uri.path.split('/')
84
+ remote_publish_path = uri.path[0, uri.path.rindex('/')]
85
+ remote_publish_dirs = remote_publish_path.split('/')
86
+ # Use array subtraction to deduce which project/version folder to upload
87
+ # to, i.e. [1,2,3] - [2,3,4] = [1]
88
+ remote_project_dirs = remote_publish_dirs - ftp_root_dirs
89
+ Net::FTP.open(uri.host) do |ftp|
90
+ ftp_login(ftp, uri.host)
91
+ # Assume that the FIG_REMOTE_URL path exists.
92
+ ftp.chdir(ftp_root_path)
93
+ remote_project_dirs.each do |dir|
94
+ # Can't automatically create parent directories, so do it manually.
95
+ if ftp.nlst().index(dir).nil?
96
+ ftp.mkdir(dir)
97
+ ftp.chdir(dir)
98
+ else
99
+ ftp.chdir(dir)
100
+ end
101
+ end
102
+ ftp.putbinaryfile(local_file)
103
+ end
104
+ end
105
+
106
+ private
107
+
108
+ def ftp_login(ftp, host)
109
+ begin
110
+ if @login
111
+ load_authentication_for host
112
+ ftp.login get_username, get_password
113
+ else
114
+ ftp.login
115
+ end
116
+ ftp.passive = true
117
+ rescue Net::FTPPermError => error
118
+ raise Fig::NetworkError.new "Could not log in: #{error.message}"
119
+ end
120
+
121
+ return
122
+ end
123
+
124
+ def download_ftp_list(uri, dirs)
125
+ # Run a bunch of these in parallel since they're slow as hell
126
+ num_threads = (ENV['FIG_FTP_THREADS'] || '16').to_i
127
+ threads = []
128
+ all_packages = []
129
+ (0..num_threads-1).each { |num| all_packages[num] = [] }
130
+ (0..num_threads-1).each do |num|
131
+ threads << Thread.new do
132
+ packages = all_packages[num]
133
+ ftp = Net::FTP.new(uri.host)
134
+ ftp_login(ftp, uri.host)
135
+ ftp.chdir(uri.path)
136
+ pos = num
137
+ while pos < dirs.length
138
+ pkg = dirs[pos]
139
+ begin
140
+ ftp.nlst(dirs[pos]).each do |ver|
141
+ packages << pkg + '/' + ver
142
+ end
143
+ rescue Net::FTPPermError
144
+ # Ignore this error because it's indicative of the FTP library
145
+ # encountering a file or directory that it does not have
146
+ # permission to open. Fig needs to be able to have secure
147
+ # repos/packages and there is no way easy way to deal with the
148
+ # permissions issues other than consuming these errors.
149
+ #
150
+ # Actually, with FTP, you can't tell the difference between a
151
+ # file not existing and not having permission to access it (which
152
+ # is probably a good thing).
153
+ end
154
+ pos += num_threads
155
+ end
156
+ ftp.close
157
+ end
158
+ end
159
+ threads.each { |thread| thread.join }
160
+ all_packages.flatten.sort
161
+ end
162
+ end
@@ -0,0 +1,61 @@
1
+ require 'net/http'
2
+ require 'uri'
3
+
4
+ require 'fig/file_not_found_error'
5
+ require 'fig/logging'
6
+ require 'fig/network_error'
7
+ require 'fig/protocol'
8
+
9
+ module Fig; end
10
+ module Fig::Protocol; end
11
+
12
+ # File transfers via HTTP.
13
+ class Fig::Protocol::HTTP
14
+ include Fig::Protocol
15
+
16
+ # Returns whether the file was not downloaded because the file already
17
+ # exists and is already up-to-date.
18
+ def download(uri, path)
19
+ log_download(uri, path)
20
+ ::File.open(path, 'wb') do |file|
21
+ file.binmode
22
+
23
+ begin
24
+ download_via_http_get(uri, file)
25
+ rescue SystemCallError => error
26
+ Fig::Logging.debug error.message
27
+ raise Fig::FileNotFoundError.new error.message, uri
28
+ rescue SocketError => error
29
+ Fig::Logging.debug error.message
30
+ raise Fig::FileNotFoundError.new error.message, uri
31
+ end
32
+ end
33
+ end
34
+
35
+ private
36
+
37
+ def download_via_http_get(uri_string, file, redirection_limit = 10)
38
+ if redirection_limit < 1
39
+ Fig::Logging.debug 'Too many HTTP redirects.'
40
+ raise Fig::FileNotFoundError.new 'Too many HTTP redirects.', uri_string
41
+ end
42
+
43
+ response = Net::HTTP.get_response(URI(uri_string))
44
+
45
+ case response
46
+ when Net::HTTPSuccess then
47
+ file.write(response.body)
48
+ when Net::HTTPRedirection then
49
+ location = response['location']
50
+ Fig::Logging.debug "Redirecting to #{location}."
51
+ download_via_http_get(location, file, redirection_limit - 1)
52
+ else
53
+ Fig::Logging.debug "Download failed: #{response.code} #{response.message}."
54
+ raise Fig::FileNotFoundError.new(
55
+ "Download failed: #{response.code} #{response.message}.", uri_string
56
+ )
57
+ end
58
+
59
+ return
60
+ end
61
+ end
@@ -0,0 +1,42 @@
1
+ require 'highline'
2
+ require 'net/netrc'
3
+
4
+ require 'fig/user_input_error'
5
+
6
+ module Fig; end
7
+ module Fig::Protocol; end
8
+
9
+ # Login information acquisition via .netrc.
10
+ module Fig::Protocol::NetRCEnabled
11
+ private
12
+
13
+ def get_username()
14
+ @username ||= HighLine.new.ask('Username: ') { |q| q.echo = true }
15
+ return @username
16
+ end
17
+
18
+ def get_password()
19
+ @password ||= HighLine.new.ask('Password: ') { |q| q.echo = false }
20
+ return @password
21
+ end
22
+
23
+ def load_authentication_for(host)
24
+ return if @username || @password
25
+
26
+ @username ||= ENV['FIG_USERNAME']
27
+ @password ||= ENV['FIG_PASSWORD']
28
+ return if @username || @password
29
+
30
+ begin
31
+ login_data = Net::Netrc.locate host
32
+ if login_data
33
+ @username = login_data.login
34
+ @password = login_data.password
35
+ end
36
+ rescue SecurityError => error
37
+ raise Fig::UserInputError.new error.message
38
+ end
39
+
40
+ return
41
+ end
42
+ end
@@ -0,0 +1,150 @@
1
+ require 'net/sftp'
2
+
3
+ require 'fig/logging'
4
+ require 'fig/network_error'
5
+ require 'fig/package_descriptor'
6
+ require 'fig/protocol'
7
+ require 'fig/protocol/netrc_enabled'
8
+
9
+ module Fig; end
10
+ module Fig::Protocol; end
11
+
12
+ # File transfers via SFTP
13
+ class Fig::Protocol::SFTP
14
+ include Fig::Protocol
15
+ include Fig::Protocol::NetRCEnabled
16
+
17
+ def download_list(uri)
18
+ package_versions = []
19
+
20
+ sftp_run(uri) do
21
+ |connection|
22
+
23
+ connection.dir.foreach uri.path do
24
+ |package_directory|
25
+
26
+ if package_directory.directory?
27
+ package_name = package_directory.name
28
+
29
+ if package_name =~ Fig::PackageDescriptor::COMPONENT_PATTERN
30
+ connection.dir.foreach "#{uri.path}/#{package_name}" do
31
+ |version_directory|
32
+
33
+ if version_directory.directory?
34
+ version_name = version_directory.name
35
+
36
+ if version_name =~ Fig::PackageDescriptor::COMPONENT_PATTERN
37
+ package_versions << "#{package_name}/#{version_name}"
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
44
+ end
45
+
46
+ return package_versions
47
+ end
48
+
49
+ # Determine whether we need to update something. Returns nil to indicate
50
+ # "don't know".
51
+ def path_up_to_date?(uri, path)
52
+ sftp_run(uri) do
53
+ |connection|
54
+
55
+ return connection.stat!(uri.path).mtime <= ::File.mtime(path)
56
+ end
57
+
58
+ return nil
59
+ end
60
+
61
+ # Returns whether the file was not downloaded because the file already
62
+ # exists and is already up-to-date.
63
+ def download(uri, path)
64
+ sftp_run(uri) do
65
+ |connection|
66
+
67
+ begin
68
+ # *sigh* Always call #stat!(), even if the local file does not exist
69
+ # because #download!() throws Strings and not proper exception objects
70
+ # when the remote path does not exist.
71
+ stat = connection.stat!(uri.path)
72
+
73
+ if ::File.exist?(path) && stat.mtime <= ::File.mtime(path)
74
+ Fig::Logging.debug "#{path} is up to date."
75
+ return false
76
+ else
77
+ log_download uri, path
78
+ connection.download! uri.path, path
79
+
80
+ return true
81
+ end
82
+ rescue Net::SFTP::StatusException => error
83
+ if error.code == Net::SFTP::Constants::StatusCodes::FX_NO_SUCH_FILE
84
+ raise Fig::FileNotFoundError.new(error.message, uri)
85
+ end
86
+ raise error
87
+ end
88
+ end
89
+
90
+ return
91
+ end
92
+
93
+ def upload(local_file, uri)
94
+ sftp_run(uri) do
95
+ |connection|
96
+
97
+ ensure_directory_exists connection, ::File.dirname(uri.path)
98
+ connection.upload! local_file, uri.path
99
+ end
100
+
101
+ return
102
+ end
103
+
104
+ private
105
+
106
+ def sftp_run(uri, &block)
107
+ host = uri.host
108
+
109
+ load_authentication_for host
110
+
111
+ begin
112
+ options = {:password => get_password}
113
+ port = uri.port
114
+ if port
115
+ options[:port] = port
116
+ end
117
+
118
+ Net::SFTP.start(host, get_username, options, &block)
119
+ rescue Net::SSH::Exception => error
120
+ raise Fig::NetworkError.new error.message
121
+ rescue Net::SFTP::Exception => error
122
+ raise Fig::NetworkError.new error.message
123
+ end
124
+
125
+ return
126
+ end
127
+
128
+ def ensure_directory_exists(connection, path)
129
+ begin
130
+ connection.lstat!(path)
131
+ return
132
+ rescue Net::SFTP::StatusException => error
133
+ if error.code != Net::SFTP::Constants::StatusCodes::FX_NO_SUCH_FILE
134
+ raise Fig::NetworkError.new(
135
+ "Could not stat #{path}: #{response.message} (#{response.code})"
136
+ )
137
+ end
138
+ end
139
+
140
+ if path == '/'
141
+ raise Fig::NetworkError.new 'Root path does not exist.'
142
+ end
143
+
144
+ ensure_directory_exists connection, ::File.dirname(path)
145
+
146
+ connection.mkdir! path
147
+
148
+ return
149
+ end
150
+ end