jets 5.0.14 → 6.0.0.beta1

data/lib/jets/event/base.rb

@@ -0,0 +1,55 @@
+require "json"
+
+# Event public methods get turned into Lambda functions.
+#
+# Jets::Event::Base < Jets::Lambda::Functions
+# Both Jets::Event::Base and Jets::Lambda::Functions have Dsl modules included.
+# So the Jets::Event::Dsl overrides some of the Jets::Lambda::Functions behavior.
+module Jets::Event
+  class Base < Jets::Lambda::Functions
+    class Error < StandardError; end
+
+    include Dsl
+
+    # non-DSL methods
+    include Helpers::KinesisEvent
+    include Helpers::LogEvent
+    include Helpers::S3Event
+    include Helpers::SnsEvent
+    include Helpers::SqsEvent
+    prepend Jets::ExceptionReporting::Process
+
+    class << self
+      include Jets::Util::Logging
+
+      def handle(event, context, meth = :perform)
+        runner = new(event, context, meth)
+        runner.send(meth)
+      end
+
+      def handle_now(meth = :handle, event = {}, context = {})
+        handle(event, context, meth)
+      end
+
+      def handle_later(meth = :handle, event = {}, context = {})
+        function = "#{name.underscore}-#{meth}" # IE: "cool_event-handle"
+        call = Jets::CLI::Call.new(
+          function: function,
+          event: JSON.dump(event),
+          invocation_type: "Event"
+        )
+        resp = begin
+          call.invoke
+        rescue Jets::CLI::Call::Error => e
+          puts "ERROR: #{e.message}".color(:red)
+          puts "The stack may not be full deployed yet.  Please check the stack and try again."
+          return
+        end
+        unless resp.status_code == 202
+          raise Error, "Error calling Lambda function #{function} with invocation_type Event. status code: #{resp.status_code}"
+        end
+        resp
+      end
+    end
+  end
+end

data/lib/jets/event/dsl/dynamodb_event.rb

@@ -0,0 +1,8 @@
+module Jets::Event::Dsl
+  module DynamodbEvent
+    # interface method
+    def dynamodb_event(table_name_without_namespace, options = {})
+      {}
+    end
+  end
+end

data/lib/jets/event/dsl/iot_event.rb

@@ -0,0 +1,17 @@
+module Jets::Event::Dsl
+  module IotEvent
+    # The user must at least pass in an SQL statement
+    # Returns topic_props
+    # interface method
+    def iot_event(props = {})
+      if props.is_a?(String) # SQL Statement
+        props = {Sql: props}
+        {TopicRulePayload: props}
+      elsif props.key?(:TopicRulePayload) # full properties structure
+        props
+      else # just the TopicRulePayload
+        {TopicRulePayload: props}
+      end
+    end
+  end
+end

data/lib/jets/event/dsl/kinesis_event.rb

@@ -0,0 +1,8 @@
+module Jets::Event::Dsl
+  module KinesisEvent
+    # interface method
+    def kinesis_event(stream_name, options = {})
+      {}
+    end
+  end
+end

data/lib/jets/event/dsl/log_event.rb

@@ -0,0 +1,8 @@
+module Jets::Event::Dsl
+  module LogEvent
+    # interface method
+    def log_event(log_group_name, props = {})
+      {}
+    end
+  end
+end

data/lib/jets/event/dsl/rate_expression.rb

@@ -0,0 +1,27 @@
+require "fugit"
+
+module Jets::Event::Dsl
+  module RateExpression
+    # normalizes the rate expression
+    def rate_expression(expr)
+      duration = Fugit::Duration.parse(expr)
+      map = {
+        sec: "second",
+        min: "minute",
+        hou: "hour",
+        day: "day",
+        wee: "week",
+        mon: "month",
+        yea: "year"
+      }
+      # duration.h has a hash like {:hou=>1}. unit is truncated to 3 characters
+      h = duration.h # IE: {:hou=>1}
+      value = h.values.first
+      unit = h.keys.first
+      unit = map[unit]
+      # Fix the unit to be singular or plural for user
+      unit = (value > 1) ? unit.pluralize : unit.singularize
+      "#{value} #{unit}"
+    end
+  end
+end

data/lib/jets/event/dsl/s3_event.rb

@@ -0,0 +1,7 @@
+module Jets::Event::Dsl
+  module S3Event
+    def s3_event(bucket_name, props = {})
+      {}
+    end
+  end
+end

data/lib/jets/event/dsl/scheduled_event.rb

@@ -0,0 +1,53 @@
+module Jets::Event::Dsl
+  module ScheduledEvent
+    include RateExpression
+    # Public: Creates CloudWatch Event Rule
+    #
+    # expression - The rate expression.
+    #
+    # Examples
+    #
+    #   rate("10 minutes")
+    #   rate("10 minutes", description: "Hard event")
+    #
+    def rate(expression, props = {})
+      expression = rate_expression(expression) # normalize the rate expression
+      md = expression.match(/\d+\s+\w+/)
+      raise ArgumentError, "Invalid rate expression: #{expression}" unless md
+
+      expression = "rate(#{expression})"
+      scheduled_event(expression, props)
+    end
+
+    # Public: Creates CloudWatch Event Rule
+    #
+    # expression - The cron expression.
+    #
+    # Examples
+    #
+    #   cron("0 */12 * * ? *")
+    #   cron("0 */12 * * ? *", description: "Hard event")
+    #
+    def cron(expression, props = {})
+      expression = normalize_cron_expression(expression)
+      scheduled_event("cron(#{expression})", props)
+    end
+
+    def normalize_cron_expression(expr)
+      parts = expr.split(" ")
+      # AWS Cron expressions require ? for the day of the week field
+      parts[-2] = "?" if parts[-2] == "*"
+      parts.join(" ")
+    end
+
+    def scheduled_event(expression, props = {})
+      props = props.merge(ScheduleExpression: expression)
+      rule_event(props)
+    end
+
+    # interface method
+    def rule_event(props = {})
+      {}
+    end
+  end
+end

data/lib/jets/event/dsl/sns_event.rb

@@ -0,0 +1,8 @@
+module Jets::Event::Dsl
+  module SnsEvent
+    # interface method
+    def sns_event(topic_name, props = {})
+      {}
+    end
+  end
+end

data/lib/jets/event/dsl/sqs_event.rb

@@ -0,0 +1,8 @@
+module Jets::Event::Dsl
+  module SqsEvent
+    # interface method
+    def sqs_event(queue_name, options = {})
+      {}
+    end
+  end
+end

data/lib/jets/{job → event}/dsl.rb

@@ -1,15 +1,15 @@
-require 'active_support'
-require 'active_support/core_ext/class'
+require "active_support"
+require "active_support/core_ext/class"
 
-# Jets::Job::Base < Jets::Lambda::Functions
-# Both Jets::Job::Base and Jets::Lambda::Functions have Dsl modules included.
-# So the Jets::Job::Dsl overrides some of the Jets::Lambda::Functions behavior.
+# Jets::Event::Base < Jets::Lambda::Functions
+# Both Jets::Event::Base and Jets::Lambda::Functions have Dsl modules included.
+# So the Jets::Event::Dsl overrides some of the Jets::Lambda::Functions behavior.
 #
 # Implements:
 #
 #   default_associated_resource_definition
 #
-module Jets::Job::Dsl
+module Jets::Event::Dsl
   extend ActiveSupport::Concern
 
   included do
@@ -17,19 +17,14 @@ module Jets::Job::Dsl
       include Jets::AwsServices
 
       include DynamodbEvent
-      include EventSourceMapping
       include IotEvent
       include KinesisEvent
       include LogEvent
-      include RuleEvent
       include S3Event
+      include ScheduledEvent
       include SnsEvent
       include SqsEvent
 
-      # Used to provide a little more identifiable event rule auto-descriptions
-      class_attribute :rule_counter
-      self.rule_counter = 0
-
       # TODO: Get rid of default_associated_resource_definition concept.
       # Also gets rid of the need to keep track of running @associated_properties too.
       def default_associated_resource_definition(meth)

data/lib/jets/{job → event}/helpers/kinesis_event.rb

@@ -1,6 +1,6 @@
-require 'base64'
+require "base64"
 
-module Jets::Job::Helpers
+module Jets::Event::Helpers
   module KinesisEvent
     def kinesis_data
       records = event["Records"]

data/lib/jets/{job → event}/helpers/log_event.rb

@@ -1,9 +1,9 @@
-require 'base64'
-require 'json'
-require 'stringio'
-require 'zlib'
+require "base64"
+require "json"
+require "stringio"
+require "zlib"
 
-module Jets::Job::Helpers
+module Jets::Event::Helpers
   module LogEvent
     def log_event
       encoded = event["awslogs"]["data"]

data/lib/jets/event/helpers/s3_event.rb

@@ -0,0 +1,76 @@
+module Jets::Event::Helpers
+  module S3Event
+    extend Memoist
+
+    def s3_events
+      encoded_messages = event[:Records].map do |record|
+        record[:Sns][:Message] # SNS message is JSON
+      end
+      # Decode the JSON messages
+      messages = encoded_messages.map do |message|
+        data = JSON.load(message)
+        ActiveSupport::HashWithIndifferentAccess.new(data)
+      end
+      # Extract the S3 event records
+      messages.map do |message|
+        message[:Records].map do |record|
+          ActiveSupport::HashWithIndifferentAccess.new(record)
+        end
+      end.flatten
+    end
+
+    def s3_events?
+      event[:Records]&.any? { |r| r.dig(:Sns, :Message) }
+    end
+
+    def s3_objects
+      s3_events.map do |record|
+        record[:s3][:object]
+      end
+    end
+
+    def s3_objects?
+      s3_events.any? { |r| r.dig(:s3, :object) }
+    end
+
+    # Downloads the s3 object and returns a Ruby File-like handle to the object
+    def s3_files
+      s3_events.map do |event|
+        bucket = event[:s3][:bucket][:name]
+        object_key = event[:s3][:object][:key]
+
+        s3 = Aws::S3::Resource.new
+        obj = s3.bucket(bucket).object(object_key)
+
+        file_path = "/tmp/s3_files/#{object_key}"
+        FileUtils.mkdir_p(File.dirname(file_path))
+        File.open(file_path, "w") do |file|
+          obj.get(response_target: file)
+        end
+
+        file = File.open(file_path, "r")
+        NamedFile.new(file, object_key)
+      end
+    end
+    memoize :s3_files
+
+    # The Ruby File handle does not store information about the filename.
+    # NamedFile includes the filename which is useful for downstream processing.
+    class NamedFile < ::File
+      extend Memoist
+      attr_reader :filename
+      alias_method :object_key, :filename
+      alias_method :key, :object_key
+
+      def initialize(file_handle, filename)
+        @filename = filename
+        super(file_handle, "r")
+      end
+
+      def content
+        read
+      end
+      memoize :content
+    end
+  end
+end

data/lib/jets/event/helpers/sns_event.rb

@@ -0,0 +1,16 @@
+module Jets::Event::Helpers
+  module SnsEvent
+    def sns_events
+      records = event["Records"]
+      return [] unless records
+      records.map do |record|
+        message = record["Sns"]["Message"]
+        ActiveSupport::HashWithIndifferentAccess.new(JSON.load(message))
+      end
+    end
+
+    def sns_events?
+      event["Records"]&.any? { |r| r.dig("Sns", "Message") }
+    end
+  end
+end

data/lib/jets/event/helpers/sqs_event.rb

@@ -0,0 +1,23 @@
+module Jets::Event::Helpers
+  module SqsEvent
+    extend Memoist
+
+    def sqs_records
+      event[:Records].map { |record| record }
+    end
+    memoize :sqs_records
+
+    def sqs_events
+      records = sqs_records
+      return [] unless records
+      records.map do |record|
+        JSON.parse(record[:body])
+      end
+    end
+    memoize :sqs_events
+
+    def sqs_events?
+      sqs_records&.any? { |r| r.dig(:body) }
+    end
+  end
+end

data/lib/jets/event/s3.rb

@@ -0,0 +1,21 @@
+module Jets::Event
+  module S3
+    extend self
+
+    # The registry tracks bucket each time an s3_event is declared
+    # Map of bucket_name => stack_name (nested part)
+    cattr_accessor :registry
+    @@registry = {}
+
+    def any?
+      !@@registry.empty?
+    end
+
+    def create_s3_event_buckets
+      buckets = @@registry.keys
+      buckets.each do |bucket|
+        Jets::AwsServices::S3Bucket.ensure_exists(bucket)
+      end
+    end
+  end
+end

data/lib/jets/git/azure.rb

@@ -0,0 +1,64 @@
+module Jets::Git
+  class Azure < Base
+    def info
+      info = {
+        git_system: "azure",
+        git_branch: git_branch,
+        git_sha: git_sha,
+        git_dirty: false
+        # git_message: nil,
+        # git_version: nil,
+      }
+      info[:git_url] = git_url if git_url
+      info
+    end
+
+    def git_branch
+      if pr_number
+        message = ENV["BUILD_SOURCEVERSIONMESSAGE"]
+        md = message.match(/Merge pull request \d+ from (.*) into (.*)/)
+        if md
+          # IE: BUILD_SOURCEVERSIONMESSAGE=Merge pull request 2 from feature into main
+          # Its a bit weird but with azure repos with check policy trigger
+          md[1]
+        else # GitHub and Bitbucket PR has actual branch though
+          # IE: SYSTEM_PULLREQUEST_SOURCEBRANCH=feature
+          message
+        end
+      else # push
+        ENV["BUILD_SOURCEBRANCHNAME"]
+      end
+    end
+
+    def git_sha
+      ENV["BUILD_SOURCEVERSION"]
+    end
+
+    def git_url
+      "#{host}/#{full_repo}"
+    end
+
+    # IE: BUILD_REPOSITORY_URI=https://tongueroo@dev.azure.com/tongueroo/infra-project/_git/infra-ci
+    def host
+      uri = URI(ENV["BUILD_REPOSITORY_URI"])
+      "#{uri.scheme}://#{uri.host}"
+    end
+
+    # IE: BUILD_REPOSITORY_URI=https://tongueroo@dev.azure.com/tongueroo/infra-project/_git/infra-ci
+    def full_repo
+      uri = URI(ENV["BUILD_REPOSITORY_URI"])
+      org = uri.path.split("/")[1] # since there's a leading /
+      repo = ENV["BUILD_REPOSITORY_NAME"] # tongueroo
+      "#{org}/#{repo}"
+    end
+
+    # IE: SYSTEM_PULLREQUEST_PULLREQUESTID=2
+    def pr_number
+      ENV["SYSTEM_PULLREQUEST_PULLREQUESTID"]
+    end
+
+    def build_type
+      ENV["SYSTEM_PULLREQUEST_PULLREQUESTID"] ? "pull_request" : "push"
+    end
+  end
+end

data/lib/jets/git/base.rb

@@ -0,0 +1,26 @@
+module Jets::Git
+  class Base
+    extend Memoist
+
+    def params
+      base.merge(info)
+    end
+    memoize :params
+
+    # interface method
+    def info
+      {}
+    end
+
+    def base
+      {
+        git_user: user.name
+      }
+    end
+
+    def user
+      User.new
+    end
+    memoize :user
+  end
+end

data/lib/jets/git/bitbucket.rb

@@ -0,0 +1,30 @@
+module Jets::Git
+  class Bitbucket < Base
+    def info
+      info = {
+        git_system: "bitbucket",
+        git_branch: git_branch,
+        git_sha: git_sha,
+        git_dirty: false
+        # git_message: nil,
+        # git_version: nil,
+      }
+      info[:git_url] = git_url if git_url
+      info
+    end
+
+    def git_branch
+      ENV["BITBUCKET_BRANCH"]
+    end
+
+    def git_sha
+      ENV["BITBUCKET_COMMIT"]
+    end
+
+    def git_url
+      host = ENV["BITBUCKET_HOST"] || "https://bitbucket.org"
+      full_repo = ENV["BITBUCKET_REPO_FULL_NAME"]
+      "#{host}/#{full_repo}"
+    end
+  end
+end

data/lib/jets/git/circleci.rb

@@ -0,0 +1,23 @@
+module Jets::Git
+  class Circleci < Base
+    def info
+      {
+        git_system: "circleci",
+        git_branch: git_branch,
+        git_sha: git_sha,
+        git_dirty: false
+        # git_message: nil,
+        # git_version: nil,
+      }
+      # info[:git_url] = git_url if git_url
+    end
+
+    def git_branch
+      ENV["CIRCLE_BRANCH"]
+    end
+
+    def git_sha
+      ENV["CIRCLE_SHA1"]
+    end
+  end
+end

data/lib/jets/git/codebuild.rb

@@ -0,0 +1,49 @@
+module Jets::Git
+  class Codebuild < Base
+    def info
+      info = {
+        git_system: "codebuild",
+        git_branch: git_branch,
+        git_sha: git_sha,
+        git_dirty: false,
+        git_url: git_url
+        # git_message: nil,
+        # git_version: nil,
+      }
+      info.delete_if { |k, v| v.nil? }
+      info
+    end
+
+    def git_branch
+      ENV["CODEBUILD_SOURCE_VERSION"]
+    end
+
+    def git_sha
+      ENV["CODEBUILD_RESOLVED_SOURCE_VERSION"]
+    end
+
+    def git_url
+      "#{host}/#{full_repo}"
+    end
+
+    def host
+      return unless ENV["CODEBUILD_SOURCE_REPO_URL"]
+      uri = URI(ENV["CODEBUILD_SOURCE_REPO_URL"]) # https://github.com/ORG/REPO
+      "#{uri.scheme}://#{uri.host}"
+    end
+
+    # ORG/REPO
+    def full_repo
+      return unless repo_url
+      uri = URI(repo_url)
+      uri.path.sub(/^\//, "")
+    end
+
+    # https://github.com/ORG/REPO
+    def repo_url
+      return unless ENV["CODEBUILD_SOURCE_REPO_URL"]
+      # https://github.com/ORG/REPO.git
+      ENV["CODEBUILD_SOURCE_REPO_URL"].sub(".git", "")
+    end
+  end
+end

data/lib/jets/git/custom.rb

@@ -0,0 +1,32 @@
+module Jets::Git
+  class Custom < Base
+    def info
+      info = {
+        git_system: "custom",
+        git_branch: git_branch,
+        git_sha: git_sha,
+        git_dirty: false,
+        git_message: git_message
+        # git_version: nil,
+      }
+      info[:git_url] = git_url if git_url
+      info
+    end
+
+    def git_branch
+      ENV["JETS_GIT_CUSTOM_BRANCH"]
+    end
+
+    def git_sha
+      ENV["JETS_GIT_CUSTOM_SHA"]
+    end
+
+    def git_url
+      ENV["JETS_GIT_CUSTOM_URL"]
+    end
+
+    def git_message
+      ENV["JETS_GIT_CUSTOM_MESSAGE"]
+    end
+  end
+end

data/lib/jets/git/git_cli.rb

@@ -0,0 +1,28 @@
+module Jets::Git
+  module GitCli
+    def git?
+      git_folder? && git_installed?
+    end
+
+    def git_folder?
+      File.exist?(".git")
+    end
+
+    def git_installed?
+      system "type git > /dev/null 2>&1"
+    end
+
+    def git(args, on_error: :nil)
+      out = `git #{args}`.strip
+      unless $?.success?
+        case on_error
+        when :raise
+          raise Jets::Git::Error, "ERROR: git #{args} failed".color(:red)
+        when :nil
+          return
+        end
+      end
+      out
+    end
+  end
+end