jets-gems 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: bcd49cdb9d4fbeb73f0bb86077b77b860b5e1c6fe73a646ad5b0983bb9d953b7
4
+ data.tar.gz: 74ddd300cf6ef1ba835202723016b70be059fcb361a8d381578f4b5b27ae8c37
5
+ SHA512:
6
+ metadata.gz: f55241adf0c2a7b46fc661ef935620b030673c9e0acb4e50bcd187943c8916ef9f199914d2e7f07a07e016d670a58765de8bd075babb506528b113504da828d6
7
+ data.tar.gz: c649a95ae12e854abb1b6076663832ceaa4508fdbe17f51412b42a20d2aee108472f888016d952e0711d896d8ff4aef94332e6e102ecb578790a323c10f24f40
@@ -0,0 +1,11 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /_yardoc/
4
+ /coverage/
5
+ /doc/
6
+ /pkg/
7
+ /spec/reports/
8
+ /tmp/
9
+
10
+ # rspec failure tracking
11
+ .rspec_status
data/.rspec ADDED
@@ -0,0 +1,3 @@
1
+ --format documentation
2
+ --color
3
+ --require spec_helper
@@ -0,0 +1,7 @@
1
+ ---
2
+ sudo: false
3
+ language: ruby
4
+ cache: bundler
5
+ rvm:
6
+ - 2.5.1
7
+ before_install: gem install bundler -v 1.16.3
@@ -0,0 +1,7 @@
1
+ # Change Log
2
+
3
+ All notable changes to this project will be documented in this file.
4
+ This project *loosely tries* to adhere to [Semantic Versioning](http://semver.org/).
5
+
6
+ ## [0.1.0]
7
+ - Initial release
data/Gemfile ADDED
@@ -0,0 +1,6 @@
1
+ source "https://rubygems.org"
2
+
3
+ git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
4
+
5
+ # Specify your gem's dependencies in jets-gems.gemspec
6
+ gemspec
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2018 Tung Nguyen
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
@@ -0,0 +1,19 @@
1
+ # Jets Gems
2
+
3
+ [Jets](http://rubyonjets.com/) is a Ruby Serverless Framework on AWS Lambda. allows you to create serverless applications with a beautiful language: Ruby. It includes everything required to build an application and deploy it to AWS Lambda. Jets makes serverless simple.
4
+
5
+ The jets-gems library works with Jets to download and extract pre-compiled gems that work on AWS Lambda.
6
+
7
+ ## Development
8
+
9
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
10
+
11
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
12
+
13
+ ## Contributing
14
+
15
+ Bug reports and pull requests are welcome on GitHub at [tongueroo/jets-gems](https://github.com/tongueroo/jets-gems)
16
+
17
+ ## License
18
+
19
+ The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
@@ -0,0 +1,6 @@
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task :default => :spec
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "jets/gems"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start(__FILE__)
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,30 @@
1
+
2
+ lib = File.expand_path("../lib", __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require "jets/gems/version"
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "jets-gems"
8
+ spec.version = Jets::Gems::VERSION
9
+ spec.authors = ["Tung Nguyen"]
10
+ spec.email = ["tongueroo@gmail.com"]
11
+
12
+ spec.summary = %q{Library manages pre-compiled Lambda Gems for Jets Ruby Serverless Framework}
13
+ spec.homepage = "https://github.com/tongueroo/jets-gems"
14
+ spec.license = "MIT"
15
+
16
+ # Specify which files should be added to the gem when it is released.
17
+ # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
18
+ spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
19
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
20
+ end
21
+ spec.bindir = "exe"
22
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
23
+ spec.require_paths = ["lib"]
24
+
25
+ spec.add_dependency "gems"
26
+
27
+ spec.add_development_dependency "bundler"
28
+ spec.add_development_dependency "rake", "~> 10.0"
29
+ spec.add_development_dependency "rspec", "~> 3.0"
30
+ end
@@ -0,0 +1 @@
1
+ require "jets/gems"
@@ -0,0 +1,17 @@
1
+ require "jets/gems/version"
2
+
3
+ module Jets
4
+ module Gems
5
+ autoload :Agree, 'jets/gems/agree'
6
+ autoload :Check, 'jets/gems/check'
7
+ autoload :Exist, 'jets/gems/exist'
8
+ autoload :Extract, 'jets/gems/extract'
9
+ autoload :Report, 'jets/gems/report'
10
+
11
+ def ruby_folder
12
+ major, minor, patch = RUBY_VERSION.split('.')
13
+ [major, minor, '0'].join('.') # 2.5.1 => 2.5.0
14
+ end
15
+ extend self
16
+ end
17
+ end
@@ -0,0 +1,60 @@
1
+ module Jets::Gems
2
+ class Agree
3
+ def initialize
4
+ @agree_file = "#{ENV['HOME']}/.jets/agree"
5
+ end
6
+
7
+ # Only prompts if hasnt prompted before and saved a ~/.jets/agree file
8
+ def prompt
9
+ return if bypass_prompt
10
+ return if File.exist?(@agree_file)
11
+
12
+ puts <<~EOL
13
+ Jets sends data about your gems to your specified lambda build service **lambdagems.com** so that it can compile and generate the necessary Lambda layers. Lambdagems only collects anonymous non-identifiable data.
14
+
15
+ Is it okay to send your gem data to Lambdagems? (Y/n)?
16
+ EOL
17
+
18
+ answer = $stdin.gets.strip
19
+ value = answer =~ /y/i ? 'yes' : 'no'
20
+
21
+ write_file(value)
22
+ end
23
+
24
+ # Allow user to bypass prompt with JETS_AGREE=1 JETS_AGREE=yes etc
25
+ # Useful for CI/CD pipelines.
26
+ def bypass_prompt
27
+ agree = ENV['JETS_AGREE']
28
+ return false unless agree
29
+
30
+ if %w[1 yes true].include?(agree.downcase)
31
+ write_file('yes')
32
+ else
33
+ write_file('no')
34
+ end
35
+
36
+ true
37
+ end
38
+
39
+ def yes?
40
+ File.exist?(@agree_file) && IO.read(@agree_file).strip == 'yes'
41
+ end
42
+
43
+ def no?
44
+ File.exist?(@agree_file) && IO.read(@agree_file).strip == 'no'
45
+ end
46
+
47
+ def yes!
48
+ write_file("yes")
49
+ end
50
+
51
+ def no!
52
+ write_file("no")
53
+ end
54
+
55
+ def write_file(content)
56
+ FileUtils.mkdir_p(File.dirname(@agree_file))
57
+ IO.write(@agree_file, content)
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,190 @@
1
+ # Assumes gems were just built and checks the filesystem to find and detect for
2
+ # compiled gems. Unless the cli option is set to true, the it'll just check
3
+ # based on the gemspecs.
4
+ module Jets::Gems
5
+ class Check
6
+ extend Memoist
7
+
8
+ attr_reader :missing_gems
9
+ def initialize(options={})
10
+ @options = options
11
+ @missing_gems = [] # keeps track of gems that are not found in any of the lambdagems sources
12
+ end
13
+
14
+ def run!
15
+ run(exit_early: true)
16
+ end
17
+
18
+ # Checks whether the gem is found on at least one of the lambdagems sources.
19
+ # By the time the loop finishes, found_gems will hold a map of gem names to found
20
+ # url sources. Example:
21
+ #
22
+ # found_gems = {
23
+ # "nokogiri-1.8.4" => "https://lambdagems.com",
24
+ # "pg-0.21.0" => "https://anothersource.com",
25
+ # }
26
+ #
27
+ def run(exit_early: false)
28
+ puts "Checking projects gems for binary Lambda gems..."
29
+ found_gems = {}
30
+ compiled_gems.each do |gem_name|
31
+ puts "Checking #{gem_name}..." if @options[:cli]
32
+ gem_exists = false
33
+ Jets.config.gems.sources.each do |source|
34
+ exist = Jets::Gems::Exist.new(source_url: source)
35
+ found = exist.check(gem_name)
36
+ # gem exists on at least of the lambdagem sources
37
+ if found
38
+ gem_exists = true
39
+ found_gems[gem_name] = source
40
+ break
41
+ end
42
+ end
43
+ unless gem_exists
44
+ @missing_gems << gem_name
45
+ end
46
+ end
47
+
48
+ if exit_early && !@missing_gems.empty?
49
+ # Exits early if not all the linux gems are available.
50
+ # Better to error now than deploy a broken package to AWS Lambda.
51
+ # Provide users with message about using their own lambdagems source.
52
+ puts missing_message
53
+ Report.missing(@missing_gems) if agree.yes?
54
+ exit 1
55
+ end
56
+
57
+ found_gems
58
+ end
59
+
60
+ def missing?
61
+ !@missing_gems.empty?
62
+ end
63
+
64
+ def missing_message
65
+ template = <<-EOL
66
+ Your project requires compiled gems were not available in any of your lambdagems sources. Unavailable pre-compiled gems:
67
+ <% missing_gems.each do |gem| %>
68
+ * <%= gem -%>
69
+ <% end %>
70
+
71
+ Your current lambdagems sources:
72
+ <% Jets.config.gems.sources.map do |source| %>
73
+ * <%= source -%>
74
+ <% end %>
75
+
76
+ Jets is unable to build a deployment package that will work on AWS Lambda without the required pre-compiled gems. To remedy this, you can:
77
+
78
+ * Use another gem that does not require compilation.
79
+ * Create your own custom layer with the gem: http://rubyonjets.com/docs/extras/custom-lambda-layers/
80
+ <% if agree.yes? -%>
81
+ * No need to report this to us, as we've already been notified.
82
+ <% elsif agree.no? -%>
83
+ * You have choosen not to report data to lambdagems so we will not be notified about these missing gems. If you change your mind you can edit ~/.jets/agree
84
+ <% end -%>
85
+
86
+ Compiled gems usually take some time to figure out how to build as they each depend on different libraries and packages.
87
+ More info: http://rubyonjets.com/docs/lambdagems/
88
+
89
+ EOL
90
+ erb = ERB.new(template, nil, '-') # trim mode https://stackoverflow.com/questions/4632879/erb-template-removing-the-trailing-line
91
+ erb.result(binding)
92
+ end
93
+
94
+ def agree
95
+ Agree.new
96
+ end
97
+ memoize :agree
98
+
99
+ # If there are subfolders compiled_gem_paths might have files deeper
100
+ # in the directory tree. So lets grab the gem name and figure out the
101
+ # unique paths of the compiled gems from there.
102
+ def compiled_gems
103
+ # @use_gemspec option finds compile gems with Gem::Specification
104
+ # The normal build process does not use this and checks the file system.
105
+ # So @use_gemspec is only used for this command:
106
+ #
107
+ # jets gems:check
108
+ #
109
+ # This is because it seems like some gems like json are remove and screws things up.
110
+ # We'll filter out for the json gem as a hacky workaround, unsure if there are more
111
+ # gems though that exhibit this behavior.
112
+ if @options[:cli]
113
+ gemspec_compiled_gems
114
+ else
115
+ compiled_gems = compiled_gem_paths.map { |p| gem_name_from_path(p) }.uniq
116
+ # Double check that the gems are also in the gemspec list since that
117
+ # one is scoped to Bundler and will only included gems used in the project.
118
+ # This handles the possiblity of stale gems leftover from previous builds
119
+ # in the cache.
120
+ # TODO: figure out if we need
121
+ # compiled_gems.select { |g| gemspec_compiled_gems.include?(g) }
122
+ end
123
+ end
124
+
125
+ # Use pre-compiled gem because the gem could have development header shared
126
+ # object file dependencies. The shared dependencies are packaged up as part
127
+ # of the pre-compiled gem so it is available in the Lambda execution environment.
128
+ #
129
+ # Example paths:
130
+ # Macosx:
131
+ # opt/ruby/gems/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/nokogiri-1.8.1
132
+ # opt/ruby/gems/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/byebug-9.1.0
133
+ # Official AWS Lambda Linux AMI:
134
+ # opt/ruby/gems/2.5.0/extensions/x86_64-linux/2.5.0-static/nokogiri-1.8.1
135
+ # Circleci Ubuntu based Linux:
136
+ # opt/ruby/gems/2.5.0/extensions/x86_64-linux/2.5.0/pg-0.21.0
137
+ def compiled_gem_paths
138
+ Dir.glob("#{Jets.build_root}/stage/opt/ruby/gems/*/extensions/**/**/*.{so,bundle}")
139
+ end
140
+
141
+ # Input: opt/ruby/gems/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/byebug-9.1.0
142
+ # Output: byebug-9.1.0
143
+ def gem_name_from_path(path)
144
+ regexp = %r{opt/ruby/gems/\d+\.\d+\.\d+/extensions/.*?/.*?/(.*?)/}
145
+ path.match(regexp)[1] # gem_name
146
+ end
147
+
148
+ # So can also check for compiled gems with Gem::Specification
149
+ # But then also includes the json gem, which then bundler removes?
150
+ # We'll figure out the the json gems.
151
+ # https://gist.github.com/tongueroo/16f4aa5ac5393424103347b0e529495e
152
+ #
153
+ # This is a faster way to check but am unsure if there are more gems than just
154
+ # json that exhibit this behavior. So only using this technique for this commmand:
155
+ #
156
+ # jets gems:check
157
+ #
158
+ # Thanks: https://gist.github.com/aelesbao/1414b169a79162b1d795 and
159
+ # https://stackoverflow.com/questions/5165950/how-do-i-get-a-list-of-gems-that-are-installed-that-have-native-extensions
160
+ def specs_with_extensions
161
+ # puts "Jets::Gems::Check#specs_with_extensions".color:red)
162
+ # puts "BUNDLE_GEMFILE #{ENV['BUNDLE_GEMFILE'].inspect}"
163
+ # puts "Jets.root #{Jets.root}"
164
+ # puts "Dir.pwd #{Dir.pwd}"
165
+ # ENV['BUNDLE_GEMFILE'] = nil
166
+ # puts "BUNDLE_GEMFILE2 #{ENV['BUNDLE_GEMFILE'].inspect}"
167
+ # gemfile = IO.readlines("#{Dir.pwd}/Gemfile")
168
+ # puts gemfile
169
+
170
+ specs = Gem::Specification.each.select { |spec| spec.extensions.any? }
171
+ specs.reject! { |spec| weird_gems.include?(spec.name) }
172
+ specs
173
+ end
174
+
175
+ def gemspec_compiled_gems
176
+ specs_with_extensions.map(&:full_name)
177
+ end
178
+
179
+ # Filter out the weird special case gems that bundler deletes?
180
+ # Probably to fix some bug.
181
+ #
182
+ # $ bundle show json
183
+ # The gem json has been deleted. It was installed at:
184
+ # /home/ec2-user/.rbenv/versions/2.5.1/lib/ruby/gems/2.5.0/gems/json-2.1.0
185
+ #
186
+ def weird_gems
187
+ %w[json]
188
+ end
189
+ end
190
+ end
@@ -0,0 +1,56 @@
1
+ require 'open-uri'
2
+
3
+ module Jets::Gems
4
+ class Exist
5
+ def initialize(options)
6
+ @options = options
7
+ end
8
+
9
+ # We check all the availability before even downloading so we can provide a
10
+ # full list of gems they might want to research all at once instead of incrementally
11
+ #
12
+ # Examples:
13
+ #
14
+ # check(single_gem)
15
+ # check(list, of, gems)
16
+ def check(*gem_names)
17
+ gem_names = gem_names.flatten
18
+ exists = gem_names.inject({}) do |hash, gem_name|
19
+ exist = url_exists?(gem_url(gem_name))
20
+ hash[gem_name] = exist
21
+ hash.merge(hash)
22
+ end
23
+
24
+ exists.values.all? {|v| v } # all_exist
25
+ end
26
+
27
+ # Example url:
28
+ # https://gems2.lambdagems.com/gems/2.5.0/byebug/byebug-9.1.0.zip
29
+ def url_exists?(url)
30
+ url = URI.parse(url)
31
+ req = Net::HTTP.new(url.host, url.port).tap do |http|
32
+ http.use_ssl = true
33
+ end
34
+ res = req.request_head(url.path)
35
+ res.code == "200"
36
+ rescue SocketError, OpenURI::HTTPError, OpenSSL::SSL::SSLError
37
+ false
38
+ end
39
+
40
+ def source_url
41
+ @options[:source_url] || s3_url
42
+ end
43
+
44
+ # gem_name: byebug-9.1.0
45
+ # Example url:
46
+ # https://gems2.lambdagems.com/gems/2.5.0/byebug/byebug-9.1.0.zip
47
+ def gem_url(gem_name)
48
+ folder = gem_name.gsub(/-(\d+\.\d+\.\d+.*)/,'') # folder: byebug
49
+ "#{source_url}/gems/#{Jets::Gems.ruby_folder}/#{folder}/#{gem_name}.zip"
50
+ end
51
+
52
+ def ruby_folder
53
+ Jets::Gems.ruby_folder
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,7 @@
1
+ module Jets::Gems
2
+ module Extract
3
+ autoload :Base, 'jets/gems/extract/base'
4
+ autoload :Gem, 'jets/gems/extract/gem'
5
+ autoload :Ruby, 'jets/gems/extract/ruby'
6
+ end
7
+ end
@@ -0,0 +1,74 @@
1
+ require "open-uri"
2
+
3
+ module Jets::Gems::Extract
4
+ class Base
5
+ class NotFound < RuntimeError; end
6
+
7
+ attr_reader :source_url
8
+ def initialize(name, options={})
9
+ @name = name
10
+ @options = options
11
+
12
+ @downloads_root = options[:downloads_root] || "/tmp/jets/#{Jets.config.project_name}/lambdagems"
13
+ @source_url = options[:source_url] || Jets.default_gems_source
14
+ end
15
+
16
+ def clean_downloads(folder)
17
+ path = "#{@downloads_root}/downloads/#{folder}"
18
+ say "Removing cache: #{path}"
19
+ FileUtils.rm_rf(path)
20
+ end
21
+
22
+ def unzip(zipfile_path, parent_folder_dest)
23
+ sh("cd #{parent_folder_dest} && unzip -qo #{zipfile_path}")
24
+ end
25
+
26
+ def sh(command)
27
+ say "=> #{command}".color(:green)
28
+ success = system(command)
29
+ abort("Command Failed") unless success
30
+ success
31
+ end
32
+
33
+ def url_exists?(url)
34
+ exist = Jets::Gems::Exist.new(@options)
35
+ exist.url_exists?(url)
36
+ end
37
+
38
+ # Returns the dest path
39
+ def download_file(source_url, dest)
40
+ say "Url #{source_url}"
41
+ return unless url_exists?(source_url)
42
+
43
+ if File.exist?(dest)
44
+ say "File already downloaded #{dest}"
45
+ return dest
46
+ end
47
+
48
+ say "Downloading..."
49
+ FileUtils.mkdir_p(File.dirname(dest)) # ensure parent folder exists
50
+
51
+ File.open(dest, 'wb') do |saved_file|
52
+ open(source_url, 'rb') do |read_file|
53
+ saved_file.write(read_file.read)
54
+ end
55
+ end
56
+ dest
57
+ end
58
+
59
+ def project_root
60
+ @options[:project_root] || "."
61
+ end
62
+
63
+ @@log_level = :info # default level is :info
64
+ # @@log_level = :debug # uncomment to debug
65
+ def log_level=(val)
66
+ @@log_level = val
67
+ end
68
+
69
+ def say(message, level=:info)
70
+ enabled = @@log_level == :debug || level == :debug
71
+ puts(message) if enabled
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,105 @@
1
+ require "gems"
2
+
3
+ # Usage:
4
+ #
5
+ # Jets::Gems::Extract::Gem.new("pg-0.21.0",
6
+ # downloads_root: cache_area, # defaults to /tmp/lambdagem
7
+ # dest: cache_area, # defaults to . (project_root)
8
+ # ).run
9
+ #
10
+ module Jets::Gems::Extract
11
+ class Gem < Base
12
+ VERSION_PATTERN = /-(\d+\.\d+.*)/
13
+
14
+ def run
15
+ say "Looking for #{full_gem_name} gem in: #{@options[:source_url]}"
16
+ clean_downloads(:gems) if @options[:clean]
17
+ zipfile_path = download_gem
18
+ remove_current_gem
19
+ unzip_file(zipfile_path)
20
+ say("Gem #{full_gem_name} unpacked at #{project_root}")
21
+ end
22
+
23
+ def unzip_file(zipfile_path)
24
+ dest = "#{Jets.build_root}/stage/opt"
25
+ say "Unpacking into #{dest}"
26
+ FileUtils.mkdir_p(dest)
27
+ unzip(zipfile_path, dest)
28
+ end
29
+
30
+ # ensure that we always have the full gem name
31
+ def full_gem_name
32
+ return @full_gem_name if @full_gem_name
33
+
34
+ if @name.match(VERSION_PATTERN)
35
+ @full_gem_name = @name
36
+ return @full_gem_name
37
+ end
38
+
39
+ # name doesnt have a version yet, so grab the latest version and add it
40
+ version = Gems.versions(@name).first
41
+ @full_gem_name = "#{@name}-#{version["number"]}"
42
+ end
43
+
44
+ def gem_name
45
+ full_gem_name.gsub(VERSION_PATTERN,'') # folder: byebug
46
+ end
47
+
48
+ # Downloads and extracts the linux gem into the proper directory.
49
+ # Extracts to: . (current directory)
50
+ #
51
+ # It produces a `bundled` folder.
52
+ # The folder contains the re-produced directory structure. Example with
53
+ # the gem: byebug-9.1.0
54
+ #
55
+ # vendor/gems/ruby/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/byebug-9.1.0
56
+ #
57
+ def download_gem
58
+ # download - also move to /tmp/jets/demo/compiled_gems folder
59
+ url = gem_url
60
+ tarball_dest = download_file(url, download_path(File.basename(url)))
61
+ unless tarball_dest
62
+ message = "Url: #{url} not found"
63
+ if @options[:exit_on_error]
64
+ say message
65
+ exit
66
+ else
67
+ raise NotFound.new(message)
68
+ end
69
+ end
70
+ say "Downloaded to: #{tarball_dest}"
71
+ tarball_dest
72
+ end
73
+
74
+ def download_path(filename)
75
+ "#{@downloads_root}/downloads/gems/#{filename}"
76
+ end
77
+
78
+ # Finds any currently install gems that matched with the gem name and version
79
+ # and remove them first.
80
+ # We clean up the current install gems first in case it was previously installed
81
+ # and has different *.so files that can be accidentally required. This
82
+ # happened with the pg gem.
83
+ def remove_current_gem
84
+ say "Removing current #{full_gem_name} gem installation:"
85
+ gem_dirs = Dir.glob("#{project_root}/**/*").select do |path|
86
+ File.directory?(path) &&
87
+ path =~ %r{vendor/gems} &&
88
+ File.basename(path) == full_gem_name
89
+ end
90
+ gem_dirs.each do |dir|
91
+ say " rm -rf #{dir}"
92
+ FileUtils.rm_rf(dir)
93
+ end
94
+ end
95
+
96
+ # full_gem_name: byebug-9.1.0
97
+ def gem_url
98
+ "#{source_url}/gems/#{ruby_folder}/#{gem_name}/#{full_gem_name}.zip"
99
+ end
100
+
101
+ def ruby_folder
102
+ Jets::Gems.ruby_folder
103
+ end
104
+ end
105
+ end
@@ -0,0 +1,71 @@
1
+ # Usage:
2
+ #
3
+ # Jets::Gems::Extract::Ruby.new("2.5.3",
4
+ # downloads_root: cache_area, # defaults to /tmp/lambdagem
5
+ # dest: cache_area, # defaults to . (project_root)
6
+ # ).run
7
+ #
8
+ module Jets::Gems::Extract
9
+ class Ruby < Base
10
+ class NotFound < RuntimeError; end
11
+
12
+ def run
13
+ say "Looking for #{full_ruby_name}"
14
+ clean_downloads(:rubies) if @options[:clean]
15
+ zip_path = download_ruby
16
+ unzip(zip_path)
17
+ end
18
+
19
+ def unzip(path)
20
+ dest = "#{Jets.build_root}/stage/code/opt"
21
+ say "Unpacking into #{dest}"
22
+ FileUtils.mkdir_p(dest)
23
+ # cd-ing dest unzips the files into that folder
24
+ sh("cd #{dest} && unzip -qo #{path}")
25
+ say("Ruby #{full_ruby_name} unziped at #{dest}", :debug)
26
+ end
27
+
28
+ def download_ruby
29
+ url = ruby_url
30
+ puts "download ruby url #{url}"
31
+ tarball_dest = download_file(url, download_path(File.basename(url)))
32
+ unless tarball_dest
33
+ message = "Url: #{url} not found"
34
+ if @options[:exit_on_error]
35
+ say message
36
+ exit
37
+ else
38
+ raise NotFound.new(message)
39
+ end
40
+ end
41
+ say "Downloaded to: #{tarball_dest}"
42
+ tarball_dest
43
+ end
44
+
45
+ def download_path(filename)
46
+ "#{@downloads_root}/downloads/rubies/#{filename}"
47
+ end
48
+
49
+ # If only the ruby version is given, then append ruby- in front. Otherwise
50
+ # leave alone.
51
+ #
52
+ # Example:
53
+ #
54
+ # 2.5.3 -> ruby-2.5.3.zip
55
+ # ruby-2.5.3 -> ruby-2.5.3.zip
56
+ # test-ruby-2.5.3 -> test-ruby-2.5.3.zip
57
+ def full_ruby_name
58
+ md = @name.match(/^(\d+\.\d+\.\d+)$/)
59
+ if md
60
+ ruby_version = md[1]
61
+ "ruby-#{ruby_version}.zip"
62
+ else
63
+ "#{@name}.zip"
64
+ end
65
+ end
66
+
67
+ def ruby_url
68
+ "#{source_url}/rubies/#{full_ruby_name}"
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,69 @@
1
+ require "net/http"
2
+
3
+ module Jets::Gems
4
+ class Report
5
+ # For local testing, example:
6
+ # LAMBDAGEM_API_URL=localhost:8888/api/v1 jets gems:check
7
+ LAMBDAGEM_API_URL = ENV["LAMBDAGEM_API_URL"] || "https://api.lambdagems.com/api/v1"
8
+
9
+ def self.missing(gems)
10
+ new(gems).report
11
+ end
12
+
13
+ def initialize(gems)
14
+ @gems = gems
15
+ end
16
+
17
+ def report
18
+ version_pattern = /(.*)-(\d+\.\d+\.\d+.*)/
19
+ threads = []
20
+ @gems.each do |gem_name|
21
+ if md = gem_name.match(version_pattern)
22
+ name, version = md[1], md[2]
23
+ threads << Thread.new do
24
+ call_api("report/missing?name=#{name}&version=#{version}", async: true)
25
+ end
26
+ else
27
+ puts "WARN: Unable to extract the version from the gem name."
28
+ end
29
+ end
30
+ # Wait for request to finish because the command might finish before
31
+ # the Threads even send the request. So we join them just case
32
+ threads.each(&:join)
33
+ end
34
+
35
+ def api_url(path)
36
+ url = "#{LAMBDAGEM_API_URL}/#{path}"
37
+ url.include?("http") ? url : "http://#{url}" # ensure http or https has been provided
38
+ end
39
+
40
+ def call_api(path, async: false)
41
+ uri = URI(api_url(path))
42
+ http = Net::HTTP.new(uri.host, uri.port)
43
+ http.use_ssl = uri.scheme == "https"
44
+ # Abusing read_timeout to mimic async fire and forget behavior.
45
+ # This makes the code continue and return very quickly and we ignore the response
46
+ # Thanks: https://www.mikeperham.com/2015/05/08/timeout-rubys-most-dangerous-api/
47
+ # https://github.com/ankane/the-ultimate-guide-to-ruby-timeouts/issues/8
48
+ http.max_retries = 0 # Fix http retries, read_timeout will cause retries immediately, we want to disable this behavior
49
+ http.read_timeout = 0.01 if async
50
+ request = Net::HTTP::Get.new(uri)
51
+ begin
52
+ response = http.request(request)
53
+ rescue Net::ReadTimeout
54
+ # Abusing read_timeout to mimic async fire and forget behavior
55
+ end
56
+ return nil if async # always return nil if async behavior requested.
57
+ # In theory we can sometimes get back a response if it returns before
58
+ # the read timeout but that's a confusing interface.
59
+
60
+ resp = {
61
+ status: response.code.to_i,
62
+ headers: response.each_header.to_h,
63
+ body: response.body,
64
+ }
65
+ # pp resp # Uncomment to debug
66
+ resp
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,5 @@
1
+ module Jets
2
+ module Gems
3
+ VERSION = "0.1.0"
4
+ end
5
+ end
metadata ADDED
@@ -0,0 +1,121 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: jets-gems
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Tung Nguyen
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2019-05-30 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: gems
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: bundler
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rake
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '10.0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '10.0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rspec
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '3.0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '3.0'
69
+ description:
70
+ email:
71
+ - tongueroo@gmail.com
72
+ executables: []
73
+ extensions: []
74
+ extra_rdoc_files: []
75
+ files:
76
+ - ".gitignore"
77
+ - ".rspec"
78
+ - ".travis.yml"
79
+ - CHANGELOG.md
80
+ - Gemfile
81
+ - LICENSE.txt
82
+ - README.md
83
+ - Rakefile
84
+ - bin/console
85
+ - bin/setup
86
+ - jets-gems.gemspec
87
+ - lib/jets-gems.rb
88
+ - lib/jets/gems.rb
89
+ - lib/jets/gems/agree.rb
90
+ - lib/jets/gems/check.rb
91
+ - lib/jets/gems/exist.rb
92
+ - lib/jets/gems/extract.rb
93
+ - lib/jets/gems/extract/base.rb
94
+ - lib/jets/gems/extract/gem.rb
95
+ - lib/jets/gems/extract/ruby.rb
96
+ - lib/jets/gems/report.rb
97
+ - lib/jets/gems/version.rb
98
+ homepage: https://github.com/tongueroo/jets-gems
99
+ licenses:
100
+ - MIT
101
+ metadata: {}
102
+ post_install_message:
103
+ rdoc_options: []
104
+ require_paths:
105
+ - lib
106
+ required_ruby_version: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
111
+ required_rubygems_version: !ruby/object:Gem::Requirement
112
+ requirements:
113
+ - - ">="
114
+ - !ruby/object:Gem::Version
115
+ version: '0'
116
+ requirements: []
117
+ rubygems_version: 3.0.3
118
+ signing_key:
119
+ specification_version: 4
120
+ summary: Library manages pre-compiled Lambda Gems for Jets Ruby Serverless Framework
121
+ test_files: []