jets-gems 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.rspec +3 -0
- data/.travis.yml +7 -0
- data/CHANGELOG.md +7 -0
- data/Gemfile +6 -0
- data/LICENSE.txt +21 -0
- data/README.md +19 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/jets-gems.gemspec +30 -0
- data/lib/jets-gems.rb +1 -0
- data/lib/jets/gems.rb +17 -0
- data/lib/jets/gems/agree.rb +60 -0
- data/lib/jets/gems/check.rb +190 -0
- data/lib/jets/gems/exist.rb +56 -0
- data/lib/jets/gems/extract.rb +7 -0
- data/lib/jets/gems/extract/base.rb +74 -0
- data/lib/jets/gems/extract/gem.rb +105 -0
- data/lib/jets/gems/extract/ruby.rb +71 -0
- data/lib/jets/gems/report.rb +69 -0
- data/lib/jets/gems/version.rb +5 -0
- metadata +121 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: bcd49cdb9d4fbeb73f0bb86077b77b860b5e1c6fe73a646ad5b0983bb9d953b7
|
4
|
+
data.tar.gz: 74ddd300cf6ef1ba835202723016b70be059fcb361a8d381578f4b5b27ae8c37
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: f55241adf0c2a7b46fc661ef935620b030673c9e0acb4e50bcd187943c8916ef9f199914d2e7f07a07e016d670a58765de8bd075babb506528b113504da828d6
|
7
|
+
data.tar.gz: c649a95ae12e854abb1b6076663832ceaa4508fdbe17f51412b42a20d2aee108472f888016d952e0711d896d8ff4aef94332e6e102ecb578790a323c10f24f40
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/CHANGELOG.md
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2018 Tung Nguyen
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
# Jets Gems
|
2
|
+
|
3
|
+
[Jets](http://rubyonjets.com/) is a Ruby Serverless Framework on AWS Lambda. allows you to create serverless applications with a beautiful language: Ruby. It includes everything required to build an application and deploy it to AWS Lambda. Jets makes serverless simple.
|
4
|
+
|
5
|
+
The jets-gems library works with Jets to download and extract pre-compiled gems that work on AWS Lambda.
|
6
|
+
|
7
|
+
## Development
|
8
|
+
|
9
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
10
|
+
|
11
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
12
|
+
|
13
|
+
## Contributing
|
14
|
+
|
15
|
+
Bug reports and pull requests are welcome on GitHub at [tongueroo/jets-gems](https://github.com/tongueroo/jets-gems)
|
16
|
+
|
17
|
+
## License
|
18
|
+
|
19
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "jets/gems"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
data/jets-gems.gemspec
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
|
2
|
+
lib = File.expand_path("../lib", __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require "jets/gems/version"
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "jets-gems"
|
8
|
+
spec.version = Jets::Gems::VERSION
|
9
|
+
spec.authors = ["Tung Nguyen"]
|
10
|
+
spec.email = ["tongueroo@gmail.com"]
|
11
|
+
|
12
|
+
spec.summary = %q{Library manages pre-compiled Lambda Gems for Jets Ruby Serverless Framework}
|
13
|
+
spec.homepage = "https://github.com/tongueroo/jets-gems"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
# Specify which files should be added to the gem when it is released.
|
17
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
18
|
+
spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
|
19
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
20
|
+
end
|
21
|
+
spec.bindir = "exe"
|
22
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
23
|
+
spec.require_paths = ["lib"]
|
24
|
+
|
25
|
+
spec.add_dependency "gems"
|
26
|
+
|
27
|
+
spec.add_development_dependency "bundler"
|
28
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
29
|
+
spec.add_development_dependency "rspec", "~> 3.0"
|
30
|
+
end
|
data/lib/jets-gems.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "jets/gems"
|
data/lib/jets/gems.rb
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
require "jets/gems/version"
|
2
|
+
|
3
|
+
module Jets
|
4
|
+
module Gems
|
5
|
+
autoload :Agree, 'jets/gems/agree'
|
6
|
+
autoload :Check, 'jets/gems/check'
|
7
|
+
autoload :Exist, 'jets/gems/exist'
|
8
|
+
autoload :Extract, 'jets/gems/extract'
|
9
|
+
autoload :Report, 'jets/gems/report'
|
10
|
+
|
11
|
+
def ruby_folder
|
12
|
+
major, minor, patch = RUBY_VERSION.split('.')
|
13
|
+
[major, minor, '0'].join('.') # 2.5.1 => 2.5.0
|
14
|
+
end
|
15
|
+
extend self
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
module Jets::Gems
|
2
|
+
class Agree
|
3
|
+
def initialize
|
4
|
+
@agree_file = "#{ENV['HOME']}/.jets/agree"
|
5
|
+
end
|
6
|
+
|
7
|
+
# Only prompts if hasnt prompted before and saved a ~/.jets/agree file
|
8
|
+
def prompt
|
9
|
+
return if bypass_prompt
|
10
|
+
return if File.exist?(@agree_file)
|
11
|
+
|
12
|
+
puts <<~EOL
|
13
|
+
Jets sends data about your gems to your specified lambda build service **lambdagems.com** so that it can compile and generate the necessary Lambda layers. Lambdagems only collects anonymous non-identifiable data.
|
14
|
+
|
15
|
+
Is it okay to send your gem data to Lambdagems? (Y/n)?
|
16
|
+
EOL
|
17
|
+
|
18
|
+
answer = $stdin.gets.strip
|
19
|
+
value = answer =~ /y/i ? 'yes' : 'no'
|
20
|
+
|
21
|
+
write_file(value)
|
22
|
+
end
|
23
|
+
|
24
|
+
# Allow user to bypass prompt with JETS_AGREE=1 JETS_AGREE=yes etc
|
25
|
+
# Useful for CI/CD pipelines.
|
26
|
+
def bypass_prompt
|
27
|
+
agree = ENV['JETS_AGREE']
|
28
|
+
return false unless agree
|
29
|
+
|
30
|
+
if %w[1 yes true].include?(agree.downcase)
|
31
|
+
write_file('yes')
|
32
|
+
else
|
33
|
+
write_file('no')
|
34
|
+
end
|
35
|
+
|
36
|
+
true
|
37
|
+
end
|
38
|
+
|
39
|
+
def yes?
|
40
|
+
File.exist?(@agree_file) && IO.read(@agree_file).strip == 'yes'
|
41
|
+
end
|
42
|
+
|
43
|
+
def no?
|
44
|
+
File.exist?(@agree_file) && IO.read(@agree_file).strip == 'no'
|
45
|
+
end
|
46
|
+
|
47
|
+
def yes!
|
48
|
+
write_file("yes")
|
49
|
+
end
|
50
|
+
|
51
|
+
def no!
|
52
|
+
write_file("no")
|
53
|
+
end
|
54
|
+
|
55
|
+
def write_file(content)
|
56
|
+
FileUtils.mkdir_p(File.dirname(@agree_file))
|
57
|
+
IO.write(@agree_file, content)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,190 @@
|
|
1
|
+
# Assumes gems were just built and checks the filesystem to find and detect for
|
2
|
+
# compiled gems. Unless the cli option is set to true, the it'll just check
|
3
|
+
# based on the gemspecs.
|
4
|
+
module Jets::Gems
|
5
|
+
class Check
|
6
|
+
extend Memoist
|
7
|
+
|
8
|
+
attr_reader :missing_gems
|
9
|
+
def initialize(options={})
|
10
|
+
@options = options
|
11
|
+
@missing_gems = [] # keeps track of gems that are not found in any of the lambdagems sources
|
12
|
+
end
|
13
|
+
|
14
|
+
def run!
|
15
|
+
run(exit_early: true)
|
16
|
+
end
|
17
|
+
|
18
|
+
# Checks whether the gem is found on at least one of the lambdagems sources.
|
19
|
+
# By the time the loop finishes, found_gems will hold a map of gem names to found
|
20
|
+
# url sources. Example:
|
21
|
+
#
|
22
|
+
# found_gems = {
|
23
|
+
# "nokogiri-1.8.4" => "https://lambdagems.com",
|
24
|
+
# "pg-0.21.0" => "https://anothersource.com",
|
25
|
+
# }
|
26
|
+
#
|
27
|
+
def run(exit_early: false)
|
28
|
+
puts "Checking projects gems for binary Lambda gems..."
|
29
|
+
found_gems = {}
|
30
|
+
compiled_gems.each do |gem_name|
|
31
|
+
puts "Checking #{gem_name}..." if @options[:cli]
|
32
|
+
gem_exists = false
|
33
|
+
Jets.config.gems.sources.each do |source|
|
34
|
+
exist = Jets::Gems::Exist.new(source_url: source)
|
35
|
+
found = exist.check(gem_name)
|
36
|
+
# gem exists on at least of the lambdagem sources
|
37
|
+
if found
|
38
|
+
gem_exists = true
|
39
|
+
found_gems[gem_name] = source
|
40
|
+
break
|
41
|
+
end
|
42
|
+
end
|
43
|
+
unless gem_exists
|
44
|
+
@missing_gems << gem_name
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
if exit_early && !@missing_gems.empty?
|
49
|
+
# Exits early if not all the linux gems are available.
|
50
|
+
# Better to error now than deploy a broken package to AWS Lambda.
|
51
|
+
# Provide users with message about using their own lambdagems source.
|
52
|
+
puts missing_message
|
53
|
+
Report.missing(@missing_gems) if agree.yes?
|
54
|
+
exit 1
|
55
|
+
end
|
56
|
+
|
57
|
+
found_gems
|
58
|
+
end
|
59
|
+
|
60
|
+
def missing?
|
61
|
+
!@missing_gems.empty?
|
62
|
+
end
|
63
|
+
|
64
|
+
def missing_message
|
65
|
+
template = <<-EOL
|
66
|
+
Your project requires compiled gems were not available in any of your lambdagems sources. Unavailable pre-compiled gems:
|
67
|
+
<% missing_gems.each do |gem| %>
|
68
|
+
* <%= gem -%>
|
69
|
+
<% end %>
|
70
|
+
|
71
|
+
Your current lambdagems sources:
|
72
|
+
<% Jets.config.gems.sources.map do |source| %>
|
73
|
+
* <%= source -%>
|
74
|
+
<% end %>
|
75
|
+
|
76
|
+
Jets is unable to build a deployment package that will work on AWS Lambda without the required pre-compiled gems. To remedy this, you can:
|
77
|
+
|
78
|
+
* Use another gem that does not require compilation.
|
79
|
+
* Create your own custom layer with the gem: http://rubyonjets.com/docs/extras/custom-lambda-layers/
|
80
|
+
<% if agree.yes? -%>
|
81
|
+
* No need to report this to us, as we've already been notified.
|
82
|
+
<% elsif agree.no? -%>
|
83
|
+
* You have choosen not to report data to lambdagems so we will not be notified about these missing gems. If you change your mind you can edit ~/.jets/agree
|
84
|
+
<% end -%>
|
85
|
+
|
86
|
+
Compiled gems usually take some time to figure out how to build as they each depend on different libraries and packages.
|
87
|
+
More info: http://rubyonjets.com/docs/lambdagems/
|
88
|
+
|
89
|
+
EOL
|
90
|
+
erb = ERB.new(template, nil, '-') # trim mode https://stackoverflow.com/questions/4632879/erb-template-removing-the-trailing-line
|
91
|
+
erb.result(binding)
|
92
|
+
end
|
93
|
+
|
94
|
+
def agree
|
95
|
+
Agree.new
|
96
|
+
end
|
97
|
+
memoize :agree
|
98
|
+
|
99
|
+
# If there are subfolders compiled_gem_paths might have files deeper
|
100
|
+
# in the directory tree. So lets grab the gem name and figure out the
|
101
|
+
# unique paths of the compiled gems from there.
|
102
|
+
def compiled_gems
|
103
|
+
# @use_gemspec option finds compile gems with Gem::Specification
|
104
|
+
# The normal build process does not use this and checks the file system.
|
105
|
+
# So @use_gemspec is only used for this command:
|
106
|
+
#
|
107
|
+
# jets gems:check
|
108
|
+
#
|
109
|
+
# This is because it seems like some gems like json are remove and screws things up.
|
110
|
+
# We'll filter out for the json gem as a hacky workaround, unsure if there are more
|
111
|
+
# gems though that exhibit this behavior.
|
112
|
+
if @options[:cli]
|
113
|
+
gemspec_compiled_gems
|
114
|
+
else
|
115
|
+
compiled_gems = compiled_gem_paths.map { |p| gem_name_from_path(p) }.uniq
|
116
|
+
# Double check that the gems are also in the gemspec list since that
|
117
|
+
# one is scoped to Bundler and will only included gems used in the project.
|
118
|
+
# This handles the possiblity of stale gems leftover from previous builds
|
119
|
+
# in the cache.
|
120
|
+
# TODO: figure out if we need
|
121
|
+
# compiled_gems.select { |g| gemspec_compiled_gems.include?(g) }
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
# Use pre-compiled gem because the gem could have development header shared
|
126
|
+
# object file dependencies. The shared dependencies are packaged up as part
|
127
|
+
# of the pre-compiled gem so it is available in the Lambda execution environment.
|
128
|
+
#
|
129
|
+
# Example paths:
|
130
|
+
# Macosx:
|
131
|
+
# opt/ruby/gems/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/nokogiri-1.8.1
|
132
|
+
# opt/ruby/gems/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/byebug-9.1.0
|
133
|
+
# Official AWS Lambda Linux AMI:
|
134
|
+
# opt/ruby/gems/2.5.0/extensions/x86_64-linux/2.5.0-static/nokogiri-1.8.1
|
135
|
+
# Circleci Ubuntu based Linux:
|
136
|
+
# opt/ruby/gems/2.5.0/extensions/x86_64-linux/2.5.0/pg-0.21.0
|
137
|
+
def compiled_gem_paths
|
138
|
+
Dir.glob("#{Jets.build_root}/stage/opt/ruby/gems/*/extensions/**/**/*.{so,bundle}")
|
139
|
+
end
|
140
|
+
|
141
|
+
# Input: opt/ruby/gems/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/byebug-9.1.0
|
142
|
+
# Output: byebug-9.1.0
|
143
|
+
def gem_name_from_path(path)
|
144
|
+
regexp = %r{opt/ruby/gems/\d+\.\d+\.\d+/extensions/.*?/.*?/(.*?)/}
|
145
|
+
path.match(regexp)[1] # gem_name
|
146
|
+
end
|
147
|
+
|
148
|
+
# So can also check for compiled gems with Gem::Specification
|
149
|
+
# But then also includes the json gem, which then bundler removes?
|
150
|
+
# We'll figure out the the json gems.
|
151
|
+
# https://gist.github.com/tongueroo/16f4aa5ac5393424103347b0e529495e
|
152
|
+
#
|
153
|
+
# This is a faster way to check but am unsure if there are more gems than just
|
154
|
+
# json that exhibit this behavior. So only using this technique for this commmand:
|
155
|
+
#
|
156
|
+
# jets gems:check
|
157
|
+
#
|
158
|
+
# Thanks: https://gist.github.com/aelesbao/1414b169a79162b1d795 and
|
159
|
+
# https://stackoverflow.com/questions/5165950/how-do-i-get-a-list-of-gems-that-are-installed-that-have-native-extensions
|
160
|
+
def specs_with_extensions
|
161
|
+
# puts "Jets::Gems::Check#specs_with_extensions".color:red)
|
162
|
+
# puts "BUNDLE_GEMFILE #{ENV['BUNDLE_GEMFILE'].inspect}"
|
163
|
+
# puts "Jets.root #{Jets.root}"
|
164
|
+
# puts "Dir.pwd #{Dir.pwd}"
|
165
|
+
# ENV['BUNDLE_GEMFILE'] = nil
|
166
|
+
# puts "BUNDLE_GEMFILE2 #{ENV['BUNDLE_GEMFILE'].inspect}"
|
167
|
+
# gemfile = IO.readlines("#{Dir.pwd}/Gemfile")
|
168
|
+
# puts gemfile
|
169
|
+
|
170
|
+
specs = Gem::Specification.each.select { |spec| spec.extensions.any? }
|
171
|
+
specs.reject! { |spec| weird_gems.include?(spec.name) }
|
172
|
+
specs
|
173
|
+
end
|
174
|
+
|
175
|
+
def gemspec_compiled_gems
|
176
|
+
specs_with_extensions.map(&:full_name)
|
177
|
+
end
|
178
|
+
|
179
|
+
# Filter out the weird special case gems that bundler deletes?
|
180
|
+
# Probably to fix some bug.
|
181
|
+
#
|
182
|
+
# $ bundle show json
|
183
|
+
# The gem json has been deleted. It was installed at:
|
184
|
+
# /home/ec2-user/.rbenv/versions/2.5.1/lib/ruby/gems/2.5.0/gems/json-2.1.0
|
185
|
+
#
|
186
|
+
def weird_gems
|
187
|
+
%w[json]
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'open-uri'
|
2
|
+
|
3
|
+
module Jets::Gems
|
4
|
+
class Exist
|
5
|
+
def initialize(options)
|
6
|
+
@options = options
|
7
|
+
end
|
8
|
+
|
9
|
+
# We check all the availability before even downloading so we can provide a
|
10
|
+
# full list of gems they might want to research all at once instead of incrementally
|
11
|
+
#
|
12
|
+
# Examples:
|
13
|
+
#
|
14
|
+
# check(single_gem)
|
15
|
+
# check(list, of, gems)
|
16
|
+
def check(*gem_names)
|
17
|
+
gem_names = gem_names.flatten
|
18
|
+
exists = gem_names.inject({}) do |hash, gem_name|
|
19
|
+
exist = url_exists?(gem_url(gem_name))
|
20
|
+
hash[gem_name] = exist
|
21
|
+
hash.merge(hash)
|
22
|
+
end
|
23
|
+
|
24
|
+
exists.values.all? {|v| v } # all_exist
|
25
|
+
end
|
26
|
+
|
27
|
+
# Example url:
|
28
|
+
# https://gems2.lambdagems.com/gems/2.5.0/byebug/byebug-9.1.0.zip
|
29
|
+
def url_exists?(url)
|
30
|
+
url = URI.parse(url)
|
31
|
+
req = Net::HTTP.new(url.host, url.port).tap do |http|
|
32
|
+
http.use_ssl = true
|
33
|
+
end
|
34
|
+
res = req.request_head(url.path)
|
35
|
+
res.code == "200"
|
36
|
+
rescue SocketError, OpenURI::HTTPError, OpenSSL::SSL::SSLError
|
37
|
+
false
|
38
|
+
end
|
39
|
+
|
40
|
+
def source_url
|
41
|
+
@options[:source_url] || s3_url
|
42
|
+
end
|
43
|
+
|
44
|
+
# gem_name: byebug-9.1.0
|
45
|
+
# Example url:
|
46
|
+
# https://gems2.lambdagems.com/gems/2.5.0/byebug/byebug-9.1.0.zip
|
47
|
+
def gem_url(gem_name)
|
48
|
+
folder = gem_name.gsub(/-(\d+\.\d+\.\d+.*)/,'') # folder: byebug
|
49
|
+
"#{source_url}/gems/#{Jets::Gems.ruby_folder}/#{folder}/#{gem_name}.zip"
|
50
|
+
end
|
51
|
+
|
52
|
+
def ruby_folder
|
53
|
+
Jets::Gems.ruby_folder
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
require "open-uri"
|
2
|
+
|
3
|
+
module Jets::Gems::Extract
|
4
|
+
class Base
|
5
|
+
class NotFound < RuntimeError; end
|
6
|
+
|
7
|
+
attr_reader :source_url
|
8
|
+
def initialize(name, options={})
|
9
|
+
@name = name
|
10
|
+
@options = options
|
11
|
+
|
12
|
+
@downloads_root = options[:downloads_root] || "/tmp/jets/#{Jets.config.project_name}/lambdagems"
|
13
|
+
@source_url = options[:source_url] || Jets.default_gems_source
|
14
|
+
end
|
15
|
+
|
16
|
+
def clean_downloads(folder)
|
17
|
+
path = "#{@downloads_root}/downloads/#{folder}"
|
18
|
+
say "Removing cache: #{path}"
|
19
|
+
FileUtils.rm_rf(path)
|
20
|
+
end
|
21
|
+
|
22
|
+
def unzip(zipfile_path, parent_folder_dest)
|
23
|
+
sh("cd #{parent_folder_dest} && unzip -qo #{zipfile_path}")
|
24
|
+
end
|
25
|
+
|
26
|
+
def sh(command)
|
27
|
+
say "=> #{command}".color(:green)
|
28
|
+
success = system(command)
|
29
|
+
abort("Command Failed") unless success
|
30
|
+
success
|
31
|
+
end
|
32
|
+
|
33
|
+
def url_exists?(url)
|
34
|
+
exist = Jets::Gems::Exist.new(@options)
|
35
|
+
exist.url_exists?(url)
|
36
|
+
end
|
37
|
+
|
38
|
+
# Returns the dest path
|
39
|
+
def download_file(source_url, dest)
|
40
|
+
say "Url #{source_url}"
|
41
|
+
return unless url_exists?(source_url)
|
42
|
+
|
43
|
+
if File.exist?(dest)
|
44
|
+
say "File already downloaded #{dest}"
|
45
|
+
return dest
|
46
|
+
end
|
47
|
+
|
48
|
+
say "Downloading..."
|
49
|
+
FileUtils.mkdir_p(File.dirname(dest)) # ensure parent folder exists
|
50
|
+
|
51
|
+
File.open(dest, 'wb') do |saved_file|
|
52
|
+
open(source_url, 'rb') do |read_file|
|
53
|
+
saved_file.write(read_file.read)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
dest
|
57
|
+
end
|
58
|
+
|
59
|
+
def project_root
|
60
|
+
@options[:project_root] || "."
|
61
|
+
end
|
62
|
+
|
63
|
+
@@log_level = :info # default level is :info
|
64
|
+
# @@log_level = :debug # uncomment to debug
|
65
|
+
def log_level=(val)
|
66
|
+
@@log_level = val
|
67
|
+
end
|
68
|
+
|
69
|
+
def say(message, level=:info)
|
70
|
+
enabled = @@log_level == :debug || level == :debug
|
71
|
+
puts(message) if enabled
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
require "gems"
|
2
|
+
|
3
|
+
# Usage:
|
4
|
+
#
|
5
|
+
# Jets::Gems::Extract::Gem.new("pg-0.21.0",
|
6
|
+
# downloads_root: cache_area, # defaults to /tmp/lambdagem
|
7
|
+
# dest: cache_area, # defaults to . (project_root)
|
8
|
+
# ).run
|
9
|
+
#
|
10
|
+
module Jets::Gems::Extract
|
11
|
+
class Gem < Base
|
12
|
+
VERSION_PATTERN = /-(\d+\.\d+.*)/
|
13
|
+
|
14
|
+
def run
|
15
|
+
say "Looking for #{full_gem_name} gem in: #{@options[:source_url]}"
|
16
|
+
clean_downloads(:gems) if @options[:clean]
|
17
|
+
zipfile_path = download_gem
|
18
|
+
remove_current_gem
|
19
|
+
unzip_file(zipfile_path)
|
20
|
+
say("Gem #{full_gem_name} unpacked at #{project_root}")
|
21
|
+
end
|
22
|
+
|
23
|
+
def unzip_file(zipfile_path)
|
24
|
+
dest = "#{Jets.build_root}/stage/opt"
|
25
|
+
say "Unpacking into #{dest}"
|
26
|
+
FileUtils.mkdir_p(dest)
|
27
|
+
unzip(zipfile_path, dest)
|
28
|
+
end
|
29
|
+
|
30
|
+
# ensure that we always have the full gem name
|
31
|
+
def full_gem_name
|
32
|
+
return @full_gem_name if @full_gem_name
|
33
|
+
|
34
|
+
if @name.match(VERSION_PATTERN)
|
35
|
+
@full_gem_name = @name
|
36
|
+
return @full_gem_name
|
37
|
+
end
|
38
|
+
|
39
|
+
# name doesnt have a version yet, so grab the latest version and add it
|
40
|
+
version = Gems.versions(@name).first
|
41
|
+
@full_gem_name = "#{@name}-#{version["number"]}"
|
42
|
+
end
|
43
|
+
|
44
|
+
def gem_name
|
45
|
+
full_gem_name.gsub(VERSION_PATTERN,'') # folder: byebug
|
46
|
+
end
|
47
|
+
|
48
|
+
# Downloads and extracts the linux gem into the proper directory.
|
49
|
+
# Extracts to: . (current directory)
|
50
|
+
#
|
51
|
+
# It produces a `bundled` folder.
|
52
|
+
# The folder contains the re-produced directory structure. Example with
|
53
|
+
# the gem: byebug-9.1.0
|
54
|
+
#
|
55
|
+
# vendor/gems/ruby/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/byebug-9.1.0
|
56
|
+
#
|
57
|
+
def download_gem
|
58
|
+
# download - also move to /tmp/jets/demo/compiled_gems folder
|
59
|
+
url = gem_url
|
60
|
+
tarball_dest = download_file(url, download_path(File.basename(url)))
|
61
|
+
unless tarball_dest
|
62
|
+
message = "Url: #{url} not found"
|
63
|
+
if @options[:exit_on_error]
|
64
|
+
say message
|
65
|
+
exit
|
66
|
+
else
|
67
|
+
raise NotFound.new(message)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
say "Downloaded to: #{tarball_dest}"
|
71
|
+
tarball_dest
|
72
|
+
end
|
73
|
+
|
74
|
+
def download_path(filename)
|
75
|
+
"#{@downloads_root}/downloads/gems/#{filename}"
|
76
|
+
end
|
77
|
+
|
78
|
+
# Finds any currently install gems that matched with the gem name and version
|
79
|
+
# and remove them first.
|
80
|
+
# We clean up the current install gems first in case it was previously installed
|
81
|
+
# and has different *.so files that can be accidentally required. This
|
82
|
+
# happened with the pg gem.
|
83
|
+
def remove_current_gem
|
84
|
+
say "Removing current #{full_gem_name} gem installation:"
|
85
|
+
gem_dirs = Dir.glob("#{project_root}/**/*").select do |path|
|
86
|
+
File.directory?(path) &&
|
87
|
+
path =~ %r{vendor/gems} &&
|
88
|
+
File.basename(path) == full_gem_name
|
89
|
+
end
|
90
|
+
gem_dirs.each do |dir|
|
91
|
+
say " rm -rf #{dir}"
|
92
|
+
FileUtils.rm_rf(dir)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
# full_gem_name: byebug-9.1.0
|
97
|
+
def gem_url
|
98
|
+
"#{source_url}/gems/#{ruby_folder}/#{gem_name}/#{full_gem_name}.zip"
|
99
|
+
end
|
100
|
+
|
101
|
+
def ruby_folder
|
102
|
+
Jets::Gems.ruby_folder
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
# Usage:
|
2
|
+
#
|
3
|
+
# Jets::Gems::Extract::Ruby.new("2.5.3",
|
4
|
+
# downloads_root: cache_area, # defaults to /tmp/lambdagem
|
5
|
+
# dest: cache_area, # defaults to . (project_root)
|
6
|
+
# ).run
|
7
|
+
#
|
8
|
+
module Jets::Gems::Extract
|
9
|
+
class Ruby < Base
|
10
|
+
class NotFound < RuntimeError; end
|
11
|
+
|
12
|
+
def run
|
13
|
+
say "Looking for #{full_ruby_name}"
|
14
|
+
clean_downloads(:rubies) if @options[:clean]
|
15
|
+
zip_path = download_ruby
|
16
|
+
unzip(zip_path)
|
17
|
+
end
|
18
|
+
|
19
|
+
def unzip(path)
|
20
|
+
dest = "#{Jets.build_root}/stage/code/opt"
|
21
|
+
say "Unpacking into #{dest}"
|
22
|
+
FileUtils.mkdir_p(dest)
|
23
|
+
# cd-ing dest unzips the files into that folder
|
24
|
+
sh("cd #{dest} && unzip -qo #{path}")
|
25
|
+
say("Ruby #{full_ruby_name} unziped at #{dest}", :debug)
|
26
|
+
end
|
27
|
+
|
28
|
+
def download_ruby
|
29
|
+
url = ruby_url
|
30
|
+
puts "download ruby url #{url}"
|
31
|
+
tarball_dest = download_file(url, download_path(File.basename(url)))
|
32
|
+
unless tarball_dest
|
33
|
+
message = "Url: #{url} not found"
|
34
|
+
if @options[:exit_on_error]
|
35
|
+
say message
|
36
|
+
exit
|
37
|
+
else
|
38
|
+
raise NotFound.new(message)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
say "Downloaded to: #{tarball_dest}"
|
42
|
+
tarball_dest
|
43
|
+
end
|
44
|
+
|
45
|
+
def download_path(filename)
|
46
|
+
"#{@downloads_root}/downloads/rubies/#{filename}"
|
47
|
+
end
|
48
|
+
|
49
|
+
# If only the ruby version is given, then append ruby- in front. Otherwise
|
50
|
+
# leave alone.
|
51
|
+
#
|
52
|
+
# Example:
|
53
|
+
#
|
54
|
+
# 2.5.3 -> ruby-2.5.3.zip
|
55
|
+
# ruby-2.5.3 -> ruby-2.5.3.zip
|
56
|
+
# test-ruby-2.5.3 -> test-ruby-2.5.3.zip
|
57
|
+
def full_ruby_name
|
58
|
+
md = @name.match(/^(\d+\.\d+\.\d+)$/)
|
59
|
+
if md
|
60
|
+
ruby_version = md[1]
|
61
|
+
"ruby-#{ruby_version}.zip"
|
62
|
+
else
|
63
|
+
"#{@name}.zip"
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def ruby_url
|
68
|
+
"#{source_url}/rubies/#{full_ruby_name}"
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
require "net/http"
|
2
|
+
|
3
|
+
module Jets::Gems
|
4
|
+
class Report
|
5
|
+
# For local testing, example:
|
6
|
+
# LAMBDAGEM_API_URL=localhost:8888/api/v1 jets gems:check
|
7
|
+
LAMBDAGEM_API_URL = ENV["LAMBDAGEM_API_URL"] || "https://api.lambdagems.com/api/v1"
|
8
|
+
|
9
|
+
def self.missing(gems)
|
10
|
+
new(gems).report
|
11
|
+
end
|
12
|
+
|
13
|
+
def initialize(gems)
|
14
|
+
@gems = gems
|
15
|
+
end
|
16
|
+
|
17
|
+
def report
|
18
|
+
version_pattern = /(.*)-(\d+\.\d+\.\d+.*)/
|
19
|
+
threads = []
|
20
|
+
@gems.each do |gem_name|
|
21
|
+
if md = gem_name.match(version_pattern)
|
22
|
+
name, version = md[1], md[2]
|
23
|
+
threads << Thread.new do
|
24
|
+
call_api("report/missing?name=#{name}&version=#{version}", async: true)
|
25
|
+
end
|
26
|
+
else
|
27
|
+
puts "WARN: Unable to extract the version from the gem name."
|
28
|
+
end
|
29
|
+
end
|
30
|
+
# Wait for request to finish because the command might finish before
|
31
|
+
# the Threads even send the request. So we join them just case
|
32
|
+
threads.each(&:join)
|
33
|
+
end
|
34
|
+
|
35
|
+
def api_url(path)
|
36
|
+
url = "#{LAMBDAGEM_API_URL}/#{path}"
|
37
|
+
url.include?("http") ? url : "http://#{url}" # ensure http or https has been provided
|
38
|
+
end
|
39
|
+
|
40
|
+
def call_api(path, async: false)
|
41
|
+
uri = URI(api_url(path))
|
42
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
43
|
+
http.use_ssl = uri.scheme == "https"
|
44
|
+
# Abusing read_timeout to mimic async fire and forget behavior.
|
45
|
+
# This makes the code continue and return very quickly and we ignore the response
|
46
|
+
# Thanks: https://www.mikeperham.com/2015/05/08/timeout-rubys-most-dangerous-api/
|
47
|
+
# https://github.com/ankane/the-ultimate-guide-to-ruby-timeouts/issues/8
|
48
|
+
http.max_retries = 0 # Fix http retries, read_timeout will cause retries immediately, we want to disable this behavior
|
49
|
+
http.read_timeout = 0.01 if async
|
50
|
+
request = Net::HTTP::Get.new(uri)
|
51
|
+
begin
|
52
|
+
response = http.request(request)
|
53
|
+
rescue Net::ReadTimeout
|
54
|
+
# Abusing read_timeout to mimic async fire and forget behavior
|
55
|
+
end
|
56
|
+
return nil if async # always return nil if async behavior requested.
|
57
|
+
# In theory we can sometimes get back a response if it returns before
|
58
|
+
# the read timeout but that's a confusing interface.
|
59
|
+
|
60
|
+
resp = {
|
61
|
+
status: response.code.to_i,
|
62
|
+
headers: response.each_header.to_h,
|
63
|
+
body: response.body,
|
64
|
+
}
|
65
|
+
# pp resp # Uncomment to debug
|
66
|
+
resp
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
metadata
ADDED
@@ -0,0 +1,121 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: jets-gems
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Tung Nguyen
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2019-05-30 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: gems
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: bundler
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '10.0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '10.0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rspec
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '3.0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '3.0'
|
69
|
+
description:
|
70
|
+
email:
|
71
|
+
- tongueroo@gmail.com
|
72
|
+
executables: []
|
73
|
+
extensions: []
|
74
|
+
extra_rdoc_files: []
|
75
|
+
files:
|
76
|
+
- ".gitignore"
|
77
|
+
- ".rspec"
|
78
|
+
- ".travis.yml"
|
79
|
+
- CHANGELOG.md
|
80
|
+
- Gemfile
|
81
|
+
- LICENSE.txt
|
82
|
+
- README.md
|
83
|
+
- Rakefile
|
84
|
+
- bin/console
|
85
|
+
- bin/setup
|
86
|
+
- jets-gems.gemspec
|
87
|
+
- lib/jets-gems.rb
|
88
|
+
- lib/jets/gems.rb
|
89
|
+
- lib/jets/gems/agree.rb
|
90
|
+
- lib/jets/gems/check.rb
|
91
|
+
- lib/jets/gems/exist.rb
|
92
|
+
- lib/jets/gems/extract.rb
|
93
|
+
- lib/jets/gems/extract/base.rb
|
94
|
+
- lib/jets/gems/extract/gem.rb
|
95
|
+
- lib/jets/gems/extract/ruby.rb
|
96
|
+
- lib/jets/gems/report.rb
|
97
|
+
- lib/jets/gems/version.rb
|
98
|
+
homepage: https://github.com/tongueroo/jets-gems
|
99
|
+
licenses:
|
100
|
+
- MIT
|
101
|
+
metadata: {}
|
102
|
+
post_install_message:
|
103
|
+
rdoc_options: []
|
104
|
+
require_paths:
|
105
|
+
- lib
|
106
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
112
|
+
requirements:
|
113
|
+
- - ">="
|
114
|
+
- !ruby/object:Gem::Version
|
115
|
+
version: '0'
|
116
|
+
requirements: []
|
117
|
+
rubygems_version: 3.0.3
|
118
|
+
signing_key:
|
119
|
+
specification_version: 4
|
120
|
+
summary: Library manages pre-compiled Lambda Gems for Jets Ruby Serverless Framework
|
121
|
+
test_files: []
|