jets 1.2.1 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +8 -0
  3. data/Gemfile.lock +4 -4
  4. data/README.md +2 -2
  5. data/lib/jets.rb +3 -4
  6. data/lib/jets/application.rb +13 -2
  7. data/lib/jets/builders.rb +3 -0
  8. data/lib/jets/builders/code_builder.rb +22 -89
  9. data/lib/jets/builders/code_size.rb +55 -0
  10. data/lib/jets/builders/gem_replacer.rb +22 -1
  11. data/lib/jets/builders/handler_generator.rb +31 -5
  12. data/lib/jets/builders/lambda_layer.rb +50 -0
  13. data/lib/jets/builders/md5.rb +10 -5
  14. data/lib/jets/builders/md5_zip.rb +1 -0
  15. data/lib/jets/builders/purger.rb +35 -0
  16. data/lib/jets/builders/rack_packager.rb +25 -5
  17. data/lib/jets/builders/rackup_wrappers/rackup +1 -5
  18. data/lib/jets/builders/ruby_packager.rb +25 -27
  19. data/lib/jets/builders/shim_vars/app.rb +3 -3
  20. data/lib/jets/builders/shim_vars/base.rb +9 -4
  21. data/lib/jets/builders/shim_vars/shared.rb +2 -2
  22. data/lib/jets/builders/templates/handler.rb +7 -0
  23. data/lib/jets/builders/tidy.rb +4 -3
  24. data/lib/jets/builders/util.rb +11 -3
  25. data/lib/jets/cfn/builders/base_child_builder.rb +6 -3
  26. data/lib/jets/cfn/builders/interface.rb +1 -1
  27. data/lib/jets/cfn/builders/parent_builder.rb +5 -0
  28. data/lib/jets/commands/build.rb +4 -2
  29. data/lib/jets/commands/help/gems/check.md +1 -1
  30. data/lib/jets/commands/main.rb +1 -1
  31. data/lib/jets/commands/new.rb +4 -2
  32. data/lib/jets/commands/sequence.rb +1 -1
  33. data/lib/jets/commands/templates/skeleton/Gemfile.tt +2 -2
  34. data/lib/jets/commands/templates/skeleton/config/application.rb.tt +0 -5
  35. data/lib/jets/commands/templates/skeleton/config/database.yml.tt +2 -2
  36. data/lib/jets/commands/templates/skeleton/config/environments/development.rb +2 -1
  37. data/lib/jets/commands/templates/skeleton/config/environments/production.rb +2 -3
  38. data/lib/jets/commands/upgrade/v1.rb +21 -0
  39. data/lib/jets/controller/renderers/template_renderer.rb +1 -1
  40. data/lib/jets/controller/rendering.rb +1 -4
  41. data/lib/jets/core.rb +45 -6
  42. data/lib/jets/internal/app/jobs/jets/preheat_job.rb +32 -34
  43. data/lib/jets/mega/request.rb +11 -0
  44. data/lib/jets/overrides/lambda.rb +1 -0
  45. data/lib/jets/overrides/lambda/marshaller.rb +31 -0
  46. data/lib/jets/overrides/rails.rb +4 -0
  47. data/lib/jets/{rails_overrides → overrides/rails}/asset_tag_helper.rb +0 -0
  48. data/lib/jets/{rails_overrides → overrides/rails}/common_methods.rb +0 -0
  49. data/lib/jets/{rails_overrides → overrides/rails}/rendering_helper.rb +0 -0
  50. data/lib/jets/{rails_overrides → overrides/rails}/url_helper.rb +0 -0
  51. data/lib/jets/poly_fun/base_executor.rb +1 -1
  52. data/lib/jets/preheat.rb +1 -0
  53. data/lib/jets/processors/main_processor.rb +3 -10
  54. data/lib/jets/{server.rb → rack_server.rb} +31 -2
  55. data/lib/jets/resource.rb +1 -1
  56. data/lib/jets/resource/api_gateway/base_path/function.rb +1 -1
  57. data/lib/jets/resource/child_stack/app_class.rb +10 -4
  58. data/lib/jets/resource/lambda.rb +5 -0
  59. data/lib/jets/resource/{function.rb → lambda/function.rb} +8 -6
  60. data/lib/jets/resource/{function → lambda/function}/environment.rb +1 -1
  61. data/lib/jets/resource/lambda/gem_layer.rb +17 -0
  62. data/lib/jets/resource/lambda/layer_version.rb +46 -0
  63. data/lib/jets/tmp_loader.rb +52 -0
  64. data/lib/jets/version.rb +1 -1
  65. data/{README → readme}/prerelease.md +0 -0
  66. data/{README → readme}/testing.md +5 -0
  67. data/vendor/jets-gems/lib/jets/gems.rb +1 -1
  68. data/vendor/jets-gems/lib/jets/gems/check.rb +7 -7
  69. data/vendor/jets-gems/lib/jets/gems/exist.rb +5 -5
  70. data/vendor/jets-gems/lib/jets/gems/extract/base.rb +4 -11
  71. data/vendor/jets-gems/lib/jets/gems/extract/gem.rb +15 -8
  72. data/vendor/jets-gems/lib/jets/gems/extract/ruby.rb +19 -10
  73. metadata +22 -15
  74. data/lib/jets/builders/templates/handler.js +0 -9
  75. data/lib/jets/builders/templates/shim.js +0 -275
  76. data/lib/jets/rails_overrides.rb +0 -4
  77. data/lib/jets/ruby_server.rb +0 -148
@@ -58,7 +58,7 @@ class Jets::Cfn::Builders
58
58
  defaults = { Type: "String" }
59
59
  options = defaults.merge(options)
60
60
  @template[:Parameters] ||= {}
61
- @template[:Parameters][name.camelize] = options
61
+ @template[:Parameters][name.to_s.camelize] = options
62
62
  end
63
63
 
64
64
  def add_outputs(attributes)
@@ -32,6 +32,11 @@ class Jets::Cfn::Builders
32
32
  resource = Jets::Resource::Iam::ApplicationRole.new
33
33
  add_resource(resource)
34
34
  add_outputs(resource.outputs)
35
+
36
+ return if Jets.poly_only?
37
+ resource = Jets::Resource::Lambda::GemLayer.new
38
+ add_resource(resource)
39
+ add_outputs(resource.outputs)
35
40
  end
36
41
 
37
42
  def build_child_resources
@@ -85,7 +85,9 @@ module Jets::Commands
85
85
  # Jets::Cfn::Builders::FunctionBuilder.new(HelloFunction)
86
86
  app_class = Jets::Klass.from_path(path)
87
87
  builder = builder_class.new(app_class)
88
- builder.build
88
+ unless Jets.poly_only? && app_class == Jets::PreheatJob
89
+ builder.build
90
+ end
89
91
  end
90
92
 
91
93
  def build_parent_template
@@ -148,7 +150,7 @@ module Jets::Commands
148
150
  has_ruby = app_files.detect do |path|
149
151
  app_class = Jets::Klass.from_path(path) # IE: PostsController, Jets::PublicController
150
152
  langs = app_class.tasks.map(&:lang)
151
- langs.include?(:ruby)
153
+ langs.include?(:ruby) && app_class != Jets::PreheatJob
152
154
  end
153
155
  !!has_ruby
154
156
  end
@@ -3,6 +3,6 @@ You can configure additional gem sources in config/application.rb:
3
3
  # Sources for check for pre-compiled Lambda gems. Checks the list in order.
4
4
  Jets.application.configure do
5
5
  config.gems.sources = [
6
- "https://gems.lambdagems.com"
6
+ "https://gems2.lambdagems.com"
7
7
  ]
8
8
  end
@@ -42,7 +42,7 @@ module Jets::Commands
42
42
  puts "=> #{command}".colorize(:green)
43
43
  puts Jets::Booter.message
44
44
  Jets::Booter.check_config_ru!
45
- Jets::Server.start(options) unless ENV['JETS_RACK'] == '0' # rack server runs in background by default
45
+ Jets::RackServer.start(options) unless ENV['JETS_RACK'] == '0' # rack server runs in background by default
46
46
  system(command)
47
47
  end
48
48
 
@@ -29,11 +29,13 @@ module Jets::Commands
29
29
  # Also had trouble unfreezing it with .dup. So using instance variables instead
30
30
  case options[:mode]
31
31
  when 'html'
32
- @webpacker = options[:webpacker]
33
32
  @bootstrap = options[:bootstrap]
33
+ @database = options[:database]
34
+ @webpacker = options[:webpacker]
34
35
  when 'api', 'job'
35
- @webpacker = false
36
36
  @bootstrap = false
37
+ @database = false
38
+ @webpacker = false
37
39
  else
38
40
  puts "Invalid mode provided: #{@options[:mode].colorize(:red)}. Please pass in an valid mode: #{VALID_MODES.join(',').colorize(:green)}."
39
41
  exit 1
@@ -56,7 +56,7 @@ private
56
56
  yarn
57
57
  public
58
58
  ]
59
- elsif !@options[:database]
59
+ elsif !@database
60
60
  # Do not even generated the config/database.yml because
61
61
  # jets webpacker:install bombs and tries to load the db since it sees a
62
62
  # config/database.yml but has there's no database pg gem configured.
@@ -6,10 +6,10 @@ gem "jets"
6
6
  # Include webpacker if you are you are building html pages
7
7
  gem "webpacker", git: "https://github.com/tongueroo/webpacker.git", branch: "jets"
8
8
  <% end -%>
9
- <% if @options[:database] == 'postgresql' %>
9
+ <% if @database == 'postgresql' %>
10
10
  # Include pg gem if you are using ActiveRecord, remove if you are not
11
11
  gem "pg", "~> 1.1.3"
12
- <% elsif @options[:database] == 'mysql' %>
12
+ <% elsif @database == 'mysql' %>
13
13
  # Include mysql2 gem if you are using ActiveRecord, remove if you are not
14
14
  gem "mysql2", "~> 0.5.2"
15
15
  <% end %>
@@ -2,11 +2,6 @@ Jets.application.configure do
2
2
  config.project_name = "<%= @project_name %>"
3
3
  config.mode = "<%= @options[:mode] %>"
4
4
 
5
- # Lazy loads ruby and gems as part of first request. Prewarming will handles
6
- # this. This also keeps the code package size small and allows us to use the
7
- # Lambda console for inline code editing.
8
- # config.ruby.lazy_load = true # default is true
9
-
10
5
  # config.prewarm.enable = true # default is true
11
6
  # config.prewarm.rate = '30 minutes' # default is '30 minutes'
12
7
  # config.prewarm.concurrency = 2 # default is 2
@@ -1,9 +1,9 @@
1
1
  default: &default
2
- adapter: <%= @options[:database] == 'mysql' ? 'mysql2' : 'postgresql' %>
2
+ adapter: <%= @database == 'mysql' ? 'mysql2' : 'postgresql' %>
3
3
  encoding: utf8
4
4
  pool: <%%= ENV["DB_POOL"] || 5 %>
5
5
  database: <%%= ENV['DB_NAME'] || '<%= @project_name %>_development' %>
6
- <% if @options[:database] == 'mysql' -%>
6
+ <% if @database == 'mysql' -%>
7
7
  username: <%%= ENV['DB_USER'] || 'root' %>
8
8
  <% else -%>
9
9
  username: <%%= ENV['DB_USER'] || ENV['USER'] %>
@@ -1,3 +1,4 @@
1
1
  Jets.application.configure do
2
- config.ruby.lazy_load = true
2
+ # Example:
3
+ # config.function.memory_size = 1536
3
4
  end
@@ -1,5 +1,4 @@
1
1
  Jets.application.configure do
2
- # If your app exceeds the AWS Lambda code size limit then Jets will automatically
3
- # enable config.ruby.lazy_load = true regardless of this setting.
4
- config.ruby.lazy_load = false
2
+ # Example:
3
+ # config.function.memory_size = 2048
5
4
  end
@@ -13,6 +13,7 @@ class Jets::Commands::Upgrade
13
13
  update_routes
14
14
  update_mode_setting
15
15
  update_config_ru
16
+ remove_ruby_lazy_load
16
17
  puts "Upgrade complete."
17
18
  end
18
19
 
@@ -79,5 +80,25 @@ class Jets::Commands::Upgrade
79
80
  puts "Update: config.ru"
80
81
  FileUtils.cp(src, dest)
81
82
  end
83
+
84
+ def remove_ruby_lazy_load
85
+ app_config = "#{Jets.root}config/application.rb"
86
+ remove_ruby_lazy_load_for(app_config)
87
+ Dir.glob("#{Jets.root}config/environments/*.rb").each do |env_config|
88
+ remove_ruby_lazy_load_for(env_config)
89
+ end
90
+ end
91
+
92
+ def remove_ruby_lazy_load_for(path)
93
+ lines = IO.readlines(path)
94
+ new_lines = lines.reject do |l|
95
+ l =~ %r{config.ruby.lazy_load}
96
+ end
97
+ return unless lines != new_lines
98
+
99
+ content = new_lines.join("")
100
+ IO.write(path, content)
101
+ puts "Update: #{path}"
102
+ end
82
103
  end
83
104
  end
@@ -117,7 +117,7 @@ module Jets::Controller::Renderers
117
117
  class << self
118
118
  def setup!
119
119
  require "action_controller"
120
- require "jets/rails_overrides"
120
+ require "jets/overrides/rails"
121
121
 
122
122
  # Load helpers
123
123
  # Assign local variable because scoe in the `:action_view do` changes
@@ -71,10 +71,7 @@ class Jets::Controller
71
71
  end
72
72
 
73
73
  def url_for(url)
74
- # No longer need to add stage name, think this is due to rack middleware support
75
- # Leaving in as comment for now just in case.
76
- # add_stage_name(url)
77
- url
74
+ add_stage_name(url)
78
75
  end
79
76
 
80
77
  def actual_host
@@ -92,7 +92,7 @@ module Jets::Core
92
92
  path =~ %r{/internal/app} ||
93
93
  path =~ %r{/jets/stack} ||
94
94
  path =~ %r{/rackup_wrappers} ||
95
- path =~ %r{/rails_overrides} ||
95
+ path =~ %r{/overrides} ||
96
96
  path =~ %r{/reconfigure_rails} ||
97
97
  path =~ %r{/templates/} ||
98
98
  path =~ %r{/version} ||
@@ -152,11 +152,6 @@ module Jets::Core
152
152
  File.exist?(path) || File.symlink?(path)
153
153
  end
154
154
 
155
- def lazy_load?
156
- return false if poly_only? # no need to lazy load when poly_only?
157
- config.ruby.lazy_load
158
- end
159
-
160
155
  def poly_only?
161
156
  return true if ENV['JETS_POLY_ONLY'] # bypass to allow rapid development of handlers
162
157
  Jets::Commands::Build.poly_only?
@@ -174,4 +169,48 @@ module Jets::Core
174
169
  def custom_domain?
175
170
  Jets.config.domain.hosted_zone_name
176
171
  end
172
+
173
+ def process(event, context, handler)
174
+ if event['_prewarm']
175
+ Jets.increase_prewarm_count
176
+ Jets.logger.info("Prewarm request")
177
+ {prewarmed_at: Time.now.to_s}
178
+ else
179
+ Jets::Processors::MainProcessor.new(event, context, handler).run
180
+ end
181
+ end
182
+
183
+ # Example: Jets.handler(self, "handlers/controllers/posts_controller.index")
184
+ def handler(lambda_context, handler)
185
+ meth = handler.split('.').last
186
+ lambda_context.send(:define_method, meth) do |event:, context:|
187
+ Jets.process(event, context, handler)
188
+ end
189
+ end
190
+
191
+ def once
192
+ boot
193
+ override_lambda_ruby_runtime
194
+ tmp_load!
195
+ start_rack_server
196
+ end
197
+
198
+ def tmp_load!
199
+ Jets::TmpLoader.load!
200
+ end
201
+
202
+ # Megamode support
203
+ def start_rack_server(options={})
204
+ rack = Jets::RackServer.new(options)
205
+ rack.start
206
+ rack.wait_for_socket
207
+ end
208
+
209
+ def default_gems_source
210
+ "https://gems2.lambdagems.com"
211
+ end
212
+
213
+ def override_lambda_ruby_runtime
214
+ require "jets/overrides/lambda"
215
+ end
177
216
  end
@@ -22,43 +22,41 @@ class Jets::PreheatJob < ApplicationJob
22
22
  }
23
23
  )
24
24
 
25
- unless Jets.poly_only?
26
- rate(PREWARM_RATE) if torching
27
- def torch
28
- threads = []
29
- CONCURRENCY.times do
30
- threads << Thread.new do
31
- # intentionally calling remote lambda for concurrency
32
- # avoid passing the _prewarm=1 flag because we want the PreheatJob#warm
33
- # to run it's normal workload.
34
- # Do not use Jets::Preheat.warm(function_name) here as that passes the _prewarm=1.
35
- function_name = "jets-preheat_job-warm"
36
- event_json = JSON.dump(event)
37
- options = call_options(event[:quiet])
38
- Jets::Commands::Call.new(function_name, event_json, options).run unless ENV['TEST']
39
- end
25
+ rate(PREWARM_RATE) if torching
26
+ def torch
27
+ threads = []
28
+ CONCURRENCY.times do
29
+ threads << Thread.new do
30
+ # intentionally calling remote lambda for concurrency
31
+ # avoid passing the _prewarm=1 flag because we want the PreheatJob#warm
32
+ # to run it's normal workload.
33
+ # Do not use Jets::Preheat.warm(function_name) here as that passes the _prewarm=1.
34
+ function_name = "jets-preheat_job-warm"
35
+ event_json = JSON.dump(event)
36
+ options = call_options(event[:quiet])
37
+ Jets::Commands::Call.new(function_name, event_json, options).run unless ENV['TEST']
40
38
  end
41
- threads.each { |t| t.join }
42
- "Finished prewarming your application with a concurrency of #{CONCURRENCY}."
43
39
  end
40
+ threads.each { |t| t.join }
41
+ "Finished prewarming your application with a concurrency of #{CONCURRENCY}."
42
+ end
44
43
 
45
- rate(PREWARM_RATE) if warming
46
- def warm
47
- options = call_options(event[:quiet])
48
- Jets::Preheat.warm_all(options)
49
- "Finished prewarming your application."
50
- end
44
+ rate(PREWARM_RATE) if warming
45
+ def warm
46
+ options = call_options(event[:quiet])
47
+ Jets::Preheat.warm_all(options)
48
+ "Finished prewarming your application."
49
+ end
51
50
 
52
- private
53
- def call_options(quiet)
54
- options = {}
55
- options.merge!(mute: true, mute_output: true) if quiet
56
- # All the methods in this Job class leads to Jets::Commands::Call.
57
- # This is true for the Jets::Preheat.warm_all also.
58
- # These jobs delegate out to Lambda function calls. We do not need/want
59
- # the invocation type: RequestResponse in this case.
60
- options.merge!(invocation_type: "Event")
61
- options
62
- end
51
+ private
52
+ def call_options(quiet)
53
+ options = {}
54
+ options.merge!(mute: true, mute_output: true) if quiet
55
+ # All the methods in this Job class leads to Jets::Commands::Call.
56
+ # This is true for the Jets::Preheat.warm_all also.
57
+ # These jobs delegate out to Lambda function calls. We do not need/want
58
+ # the invocation type: RequestResponse in this case.
59
+ options.merge!(invocation_type: "Event")
60
+ options
63
61
  end
64
62
  end
@@ -4,6 +4,7 @@ require 'rack'
4
4
  module Jets::Mega
5
5
  class Request
6
6
  autoload :Source, 'jets/mega/request/source'
7
+ JETS_OUTPUT = "/tmp/jets-output.log"
7
8
 
8
9
  extend Memoist
9
10
 
@@ -45,6 +46,8 @@ module Jets::Mega
45
46
  # Make request
46
47
  response = http.request(request)
47
48
 
49
+ puts_rack_output
50
+
48
51
  {
49
52
  status: response.code.to_i,
50
53
  headers: response.each_header.to_h,
@@ -52,6 +55,14 @@ module Jets::Mega
52
55
  }
53
56
  end
54
57
 
58
+ # Grab the rack output from the /tmp/jets-output.log and puts it back in the
59
+ # main process' stdout
60
+ def puts_rack_output
61
+ return unless File.exist?(JETS_OUTPUT)
62
+ puts IO.readlines(JETS_OUTPUT)
63
+ File.truncate(JETS_OUTPUT, 0)
64
+ end
65
+
55
66
  def get_uri
56
67
  url = "http://localhost:9292#{@controller.request.path}" # local rack server
57
68
  unless @controller.query_parameters.empty?
@@ -0,0 +1 @@
1
+ require "jets/overrides/lambda/marshaller"
@@ -0,0 +1,31 @@
1
+ require 'json'
2
+
3
+ # Hack AwsLambda Ruby Runtime to fix .to_json issue collision with ActiveSupport.
4
+ # To reproduce:
5
+ # Create a shared resource from the docs and call sns.publish
6
+ #
7
+ # Causes an infinite loop when calling sns.publish somehow.
8
+ # Overriding with JSON.dump and follow up with AWS ticket.
9
+ module AwsLambda
10
+ class Marshaller
11
+ class << self
12
+ # By default, just runs #to_json on the method's response value.
13
+ # This can be overwritten by users who know what they are doing.
14
+ # The response is an array of response, content-type.
15
+ # If returned without a content-type, it is assumed to be application/json
16
+ # Finally, StringIO/IO is used to signal a response that shouldn't be
17
+ # formatted as JSON, and should get a different content-type header.
18
+ def marshall_response(method_response)
19
+ case method_response
20
+ when StringIO, IO
21
+ [method_response, 'application/unknown']
22
+ else
23
+ # Orignal method calls .to_json but this collides with ActiveSupport's to_json
24
+ # method_response.to_json # application/json is assumed
25
+ JSON.dump(method_response)
26
+ end
27
+ end
28
+
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,4 @@
1
+ require "jets/overrides/rails/common_methods"
2
+ require "jets/overrides/rails/url_helper"
3
+ require "jets/overrides/rails/rendering_helper"
4
+ require "jets/overrides/rails/asset_tag_helper"
@@ -116,7 +116,7 @@ class Jets::PolyFun
116
116
  def handler
117
117
  # Must use the generated CloudFormation template to get the handler because
118
118
  # the handler is derived from mutiple sources.
119
- resource = Jets::Resource::Function.new(@task)
119
+ resource = Jets::Resource::Lambda::Function.new(@task)
120
120
  full_handler = resource.properties["Handler"] # full handler here
121
121
  File.extname(full_handler).sub(/^./,'') # the extension of the full handler is the handler
122
122
  end
@@ -18,6 +18,7 @@ module Jets
18
18
  def initialize(options)
19
19
  @options = options # passed to Call.new options
20
20
  @options[:mute_output] = true if @options[:mute_output].nil?
21
+ @options[:lambda_proxy] = false # do not transform controller event from {"event": "1"} to {"queryStringParameters":{"_prewarm":"1"}}
21
22
  end
22
23
 
23
24
  # Makes remote call to the Lambda function.
@@ -5,9 +5,8 @@ require 'stringio'
5
5
  class Jets::Processors::MainProcessor
6
6
  attr_reader :event, :context, :handler
7
7
  def initialize(event, context, handler)
8
- # assume valid json from Lambda
9
- @event = JSON.parse(event)
10
- @context = JSON.parse(context)
8
+ @event = event
9
+ @context = context
11
10
  @handler = handler
12
11
  end
13
12
 
@@ -38,13 +37,7 @@ class Jets::Processors::MainProcessor
38
37
  result["headers"]["x-jets-prewarm-count"] = Jets.prewarm_count
39
38
  end
40
39
 
41
- # Puts the return value of project code to stdout because this is
42
- # what eventually gets used by API Gateway.
43
- # Explicitly using $stdout since puts has been redirected to $stderr.
44
- #
45
- # JSON.dump is pretty robust. If it cannot dump the structure into a
46
- # json string, it just dumps it to a plain text string.
47
- Jets::Util.normalize_result(result) # resp is a String
40
+ result
48
41
  rescue Exception => e
49
42
  unless ENV['TEST']
50
43
  # Customize error message slightly so nodejs shim can process the