envirobly 0.7.2 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a1ffc05f7ea0f1e40889fa8a817bd44f9cf9fc3ee4d7197e48f46bd17706b017
-  data.tar.gz: 053373d7435caa5b10d301afafd3ef76777f29bffb59c108cd6ef4399d666061
+  metadata.gz: 9ef94e7e6db204bc73126d41dae8d3417e4ea7ac35e7cbecd5d496dcee842ee3
+  data.tar.gz: 32f53215e81d8e3d984cd7caf58fc0944c9636ba587e59dbd4573bfcd04f8a08
 SHA512:
-  metadata.gz: f0d071e480b64d91210ae0eb422d29707950c3288ca7e8a85354bfbaff1e78bec44e0bb88606edf264aab94bf2478e85fd6a85fba1f920e2b028c55c457bdc3b
-  data.tar.gz: ee38e1c77c0b66b90d4b36d3c89a217caef7f71f1e9c34d32ac4731eba898b265519e018501a5d5ebe41f644ca5e8534e04cf07b4ef5afa592595f60ddd5f453
+  metadata.gz: ef5d512c3dab84e7f4cc5ce45804d30cce526b61c83407895952536a45a901715aee2ceb39b71e5b8d68d8e477f20b4628f9f94df41a932f603a511e3d9fface
+  data.tar.gz: d5444e2eb995be568dcc57e7df33145d2b17792f212c9ed605a3c456d511f0e2b80968d4900642c3483d9eb01afb550e1b0071f212877706a8e7a9e265d87a9e

data/lib/envirobly/access_token.rb

@@ -2,35 +2,86 @@ require "fileutils"
 require "pathname"
 
 class Envirobly::AccessToken
-  def initialize(token = ENV.fetch("ENVIROBLY_ACCESS_TOKEN", nil))
-    if token.nil? && File.exist?(access_token_path)
-      @token = File.read(access_token_path)
+  include Envirobly::Colorize
+
+  attr_reader :shell
+
+  class << self
+    def destroy
+      if File.exist?(path)
+        FileUtils.rm path
+      end
+    end
+
+    def dir
+      if ENV["XDG_CONFIG_HOME"]
+        Pathname.new(ENV["XDG_CONFIG_HOME"]).join("envirobly")
+      else
+        Pathname.new(Dir.home).join(".envirobly")
+      end
+    end
+
+    def path
+      dir.join "access_token"
+    end
+  end
+
+  def initialize(token = ENV["ENVIROBLY_ACCESS_TOKEN"].presence, shell: nil)
+    @shell = shell
+
+    if token.blank? && File.exist?(self.class.path)
+      @token = File.read(self.class.path)
     else
       @token = token
     end
   end
 
   def save
-    FileUtils.mkdir_p config_root
-    File.write access_token_path, @token
-    File.chmod 0600, access_token_path
-    puts "Access token saved to #{access_token_path}"
+    FileUtils.mkdir_p self.class.dir
+    File.write self.class.path, @token
+    File.chmod 0600, self.class.path
   end
 
   def as_http_bearer
     "Bearer #{@token}"
   end
 
-  private
-    def config_root
-      if ENV["XDG_CONFIG_HOME"]
-        Pathname.new(ENV["XDG_CONFIG_HOME"]).join("envirobly")
-      else
-        Pathname.new(Dir.home).join(".envirobly")
+  def require!
+    return if @token.present?
+
+    shell.say "This action requires you to be signed in."
+    shell.say "Please visit https://on.envirobly.com/profile/access_tokens"
+    shell.say "to generate an access token and then paste it in here."
+    shell.say
+
+    set
+  end
+
+  def set
+    @token = nil
+
+    while @token.blank?
+      begin
+        @token = shell.ask("Access Token:", echo: false)
+      rescue Interrupt
+        shell.say
+        shell.say_error "Cancelled"
+        exit
       end
-    end
 
-    def access_token_path
-      config_root.join "access_token"
+      api = Envirobly::Api.new(access_token: self)
+
+      # TODO: Eventually replace with custom `whoami` API that returns name, email...
+      if api.list_accounts.success?
+        save
+        shell.say
+        shell.say "Successfully signed in "
+        shell.say green_check
+      else
+        shell.say
+        shell.say_error "This token is invalid. Please try again"
+        @token = nil
+      end
     end
+  end
 end

data/lib/envirobly/api.rb

@@ -4,36 +4,54 @@ require "socket"
 require "uri"
 
 class Envirobly::Api
-  HOST = ENV["ENVIROBLY_API_HOST"] || "envirobly.com"
+  HOST = ENV["ENVIROBLY_API_HOST"].presence || "on.envirobly.com"
   USER_AGENT = "Envirobly CLI v#{Envirobly::VERSION}"
   CONTENT_TYPE = "application/json"
 
-  def initialize
-    @access_token = Envirobly::AccessToken.new
+  def initialize(access_token: Envirobly::AccessToken.new)
+    @access_token = access_token
   end
 
-  def create_deployment(params)
-    post_as_json(api_v1_deployments_url, params:, headers: authorization_headers).tap do |response|
-      unless response.code.to_i == 200
-        $stderr.puts "Deployment creation request responded with #{response.code}. Aborting."
+  def validate_shape(params)
+    post_as_json(api_v1_shape_validations_url, params:, headers: authorization_headers).tap do |response|
+      unless response.success?
+        $stderr.puts "Validation request responded with #{response.code}. Aborting."
         exit 1
       end
     end
   end
 
-  RETRY_INTERVAL_SECONDS = 3
-  MAX_RETRIES = 5
+  def create_deployment(params)
+    post_as_json(api_v1_deployments_url, params:, headers: authorization_headers)
+  end
+
+  def list_accounts
+    get_as_json api_v1_accounts_url, headers: authorization_headers
+  end
+
+  def list_regions
+    get_as_json api_v1_regions_url, headers: authorization_headers
+  end
+
+  MAX_RETRIES = 30
+  SHORT_RETRY_INTERVAL = 2.seconds
+  LONG_RETRY_INTERVAL = 6.seconds
   def get_deployment_with_delay_and_retry(url, tries = 1)
-    sleep RETRY_INTERVAL_SECONDS * tries
+    sleep SHORT_RETRY_INTERVAL * tries
     response = get_as_json URI(url)
 
-    if response.code.to_i == 200
+    if response.success?
       response
     elsif MAX_RETRIES <= tries
       $stderr.puts "Max retries exhausted while waiting for deployment credentials. Aborting."
       exit 1
     else
-      sleep RETRY_INTERVAL_SECONDS * tries
+      if tries > 3
+        sleep LONG_RETRY_INTERVAL
+      else
+        sleep SHORT_RETRY_INTERVAL
+      end
+
       get_deployment_with_delay_and_retry(url, tries + 1)
     end
   end
@@ -55,8 +73,24 @@ class Envirobly::Api
   end
 
   private
+    def api_v1_shape_validations_url
+      api_url_for "v1/shape_validations"
+    end
+
     def api_v1_deployments_url
-      URI::HTTPS.build(host: HOST, path: "/api/v1/deployments")
+      api_url_for "v1/deployments"
+    end
+
+    def api_v1_accounts_url
+      api_url_for "v1/accounts"
+    end
+
+    def api_v1_regions_url
+      api_url_for "v1/regions"
+    end
+
+    def api_url_for(path)
+      URI::HTTPS.build(host: HOST, path: "/api/#{path}")
     end
 
     def request(url, type:, headers: {})
@@ -74,7 +108,11 @@ class Envirobly::Api
 
       http.request(request).tap do |response|
         def response.object
-          @json_parsed_body ||= JSON.parse body
+          @json_parsed_body ||= JSON.parse(body)
+        end
+
+        def response.success?
+          (200..299).include?(code.to_i)
         end
       end
     end

data/lib/envirobly/aws/credentials.rb

@@ -1,8 +1,13 @@
+# @deprecated
 class Envirobly::Aws::Credentials
   def initialize(params)
     @params = params
   end
 
+  def to_h
+    @params
+  end
+
   def as_env_vars
     [
       %(AWS_ACCESS_KEY_ID="#{@params.fetch("access_key_id")}"),

data/lib/envirobly/aws/s3.rb

@@ -0,0 +1,221 @@
+require "zlib"
+require "json"
+require "open3"
+require "concurrent"
+require "aws-sdk-s3"
+
+class Envirobly::Aws::S3
+  OBJECTS_PREFIX = "blobs"
+  MANIFESTS_PREFIX = "manifests"
+  CONCURRENCY = 6
+
+  def initialize(bucket:, region:, credentials: nil)
+    @region = region
+    @bucket = bucket
+
+    client_options = { region: }
+    unless credentials.nil?
+      client_options.merge! credentials.transform_keys(&:to_sym)
+    end
+
+    @client = Aws::S3::Client.new(client_options)
+    resource = Aws::S3::Resource.new(client: @client)
+    @bucket_resource = resource.bucket(@bucket)
+  end
+
+  def push(commit)
+    if object_exists?(manifest_key(commit.object_tree_checksum))
+      print "Build context is already uploaded"
+      $stdout.flush
+      return
+    end
+
+    # puts "Pushing #{commit.object_tree_checksum} to #{@bucket}"
+
+    manifest = []
+    objects_count = 0
+    objects_to_upload = []
+    remote_object_hashes = list_object_hashes
+
+    commit.object_tree.each do |chdir, objects|
+      objects.each do |(mode, type, object_hash, path)|
+        objects_count += 1
+        path = File.join chdir.delete_prefix(commit.working_dir), path
+        manifest << [ mode, type, object_hash, path.delete_prefix("/") ]
+
+        next if remote_object_hashes.include?(object_hash)
+        objects_to_upload << [ chdir, object_hash ]
+      end
+    end
+
+    upload_git_objects(objects_to_upload)
+    upload_manifest manifest_key(commit.object_tree_checksum), manifest
+  end
+
+  def pull(object_tree_checksum, target_dir)
+    puts "Pulling #{object_tree_checksum} into #{target_dir}"
+
+    manifest = fetch_manifest(object_tree_checksum)
+    FileUtils.mkdir_p(target_dir)
+
+    puts "Downloading #{manifest.size} files"
+    pool = Concurrent::FixedThreadPool.new(CONCURRENCY)
+
+    manifest.each do |(mode, type, object_hash, path)|
+      pool.post do
+        target_path = File.join target_dir, path
+
+        if mode == Envirobly::Git::Commit::SYMLINK_FILE_MODE
+          fetch_symlink(object_hash, target_path:)
+        else
+          fetch_object(object_hash, target_path:)
+
+          if mode == Envirobly::Git::Commit::EXECUTABLE_FILE_MODE
+            FileUtils.chmod("+x", target_path)
+          end
+        end
+      end
+    end
+
+    pool.shutdown
+    pool.wait_for_termination
+  end
+
+  private
+    def list_object_hashes
+      @client.list_objects({
+        bucket: @bucket,
+        prefix: "#{OBJECTS_PREFIX}/"
+      }).map do |response|
+        response.contents.map do |object|
+          object.key.delete_prefix("#{OBJECTS_PREFIX}/").delete_suffix(".gz")
+        end
+      end.flatten
+    end
+
+    def object_key(object_hash)
+      "#{OBJECTS_PREFIX}/#{object_hash}.gz"
+    end
+
+    def manifest_key(object_tree_checksum)
+      "#{MANIFESTS_PREFIX}/#{object_tree_checksum}.gz"
+    end
+
+    def object_exists?(key)
+      @client.head_object(bucket: @bucket, key:)
+      true
+    rescue Aws::S3::Errors::NotFound
+      false
+    end
+
+    def compress_and_upload_object(object_hash, chdir:)
+      key = object_key object_hash
+
+      Tempfile.create([ "envirobly-push", ".gz" ]) do |tempfile|
+        gz = Zlib::GzipWriter.new(tempfile)
+
+        Open3.popen3("git", "cat-file", "-p", object_hash, chdir:) do |_, stdout, stderr, thread|
+          IO.copy_stream(stdout, gz)
+
+          unless thread.value.success?
+            raise "`git cat-file -p #{object_hash}` failed: #{stderr.read}"
+          end
+        ensure
+          gz.close
+        end
+
+        @client.put_object(bucket: @bucket, body: tempfile, key:)
+
+        # puts "⤴ #{key}"
+      end
+    end
+
+    def upload_git_objects(objects)
+      pool = Concurrent::FixedThreadPool.new(CONCURRENCY)
+      uploaded = Concurrent::AtomicFixnum.new
+      objects_count = objects.count
+
+      notifier = Thread.new do
+        next unless objects_count > 0
+
+        # Hide cursor
+        # print "\e[?25l"
+        # $stdout.flush
+
+        loop do
+          value = uploaded.value
+          print "\rUploading build context files: #{value}/#{objects_count}"
+          $stdout.flush
+          sleep 0.5
+          break if value >= objects_count
+        end
+
+        # Show cursor again
+        # print "\e[?25h\n"
+      end
+
+      objects.each do |(chdir, object_hash)|
+        pool.post do
+          compress_and_upload_object(object_hash, chdir:)
+          uploaded.increment
+        end
+      end
+
+      pool.shutdown
+      pool.wait_for_termination
+      notifier.join
+    end
+
+    def upload_manifest(key, content)
+      Tempfile.create([ "envirobly-push", ".gz" ]) do |tempfile|
+        gz = Zlib::GzipWriter.new(tempfile)
+        gz.write JSON.dump(content)
+        gz.close
+
+        @client.put_object(bucket: @bucket, body: tempfile, key:)
+
+        # puts "⤴ #{key}"
+      end
+    end
+
+    def fetch_manifest(ref)
+      stream = @bucket_resource.object(manifest_key(ref)).get.body
+      JSON.parse Zlib::GzipReader.new(stream).read
+    rescue Aws::S3::Errors::NoSuchKey
+      puts "Commit #{ref} doesn't exist at s3://#{@bucket}"
+      exit 1
+    end
+
+    def fetch_object(object_hash, target_path:)
+      FileUtils.mkdir_p File.dirname(target_path)
+
+      key = object_key object_hash
+      stream = @bucket_resource.object(key).get.body
+
+      File.open(target_path, "wb") do |target|
+        gz = Zlib::GzipReader.new(stream)
+        IO.copy_stream(gz, target)
+        gz.close
+      end
+    end
+
+    def fetch_symlink(object_hash, target_path:)
+      FileUtils.mkdir_p File.dirname(target_path)
+
+      key = object_key object_hash
+      gz = Zlib::GzipReader.new @bucket_resource.object(key).get.body
+      symlink_to = gz.read
+      gz.close
+
+      FileUtils.ln_s symlink_to, target_path
+    end
+
+    def format_duration(duration)
+      total_seconds = duration.to_i
+      minutes = (total_seconds / 60).floor
+      seconds = (total_seconds % 60).ceil
+      result = [ "#{seconds}s" ]
+      result.prepend "#{minutes}m" if minutes > 0
+      result.join " "
+    end
+end

data/lib/envirobly/cli/main.rb

@@ -1,55 +1,107 @@
 class Envirobly::Cli::Main < Envirobly::Base
+  include Envirobly::Colorize
+
   desc "version", "Show Envirobly CLI version"
+  method_option :pure, type: :boolean, default: false
   def version
-    puts Envirobly::VERSION
+    if options.pure
+      puts Envirobly::VERSION
+    else
+      puts "envirobly CLI v#{Envirobly::VERSION}"
+    end
+  end
+
+  desc "signin", "Set access token generated at Envirobly"
+  def signin
+    access_token = Envirobly::AccessToken.new(shell:)
+    access_token.set
+  end
+
+  desc "signout", "Sign out"
+  def signout
+    Envirobly::AccessToken.destroy
+    say "You've signed out."
+    say "This didn't delete the access token itself."
+    say "You can sign in again with `envirobly signin`."
+  end
+
+  desc "set_default_account", "Choose default account to deploy the current project to"
+  def set_default_account
+    Envirobly::Defaults::Account.new(shell:).require_id
+  end
+
+  desc "set_default_region", "Set default region for the current project when deploying for the first time"
+  def set_default_region
+    Envirobly::Defaults::Region.new(shell:).require_id
   end
 
   desc "validate", "Validates config"
   def validate
-    commit = Envirobly::Git::Unstaged.new
-    config = Envirobly::Config.new(commit)
-    config.validate
-
-    if config.errors.any?
-      puts "Issues found validating `#{Envirobly::Config::PATH}`:"
-      puts
-      config.errors.each_with_index do |error, index|
-        puts "  #{index + 1}. #{error}"
-      end
-      puts
-      exit 1
+    Envirobly::AccessToken.new(shell:).require!
+
+    configs = Envirobly::Config.new
+    api = Envirobly::Api.new
+
+    params = { validation: configs.to_params }
+    response = api.validate_shape params
+
+    if response.object.fetch("valid")
+      puts "Config is valid #{green_check}"
     else
-      puts "All checks pass."
+      display_config_errors response.object.fetch("errors")
+      exit 1
     end
   end
 
-  desc "deploy ENVIRONMENT", "Deploy to environment identified by name or URL"
+  desc "deploy [ENVIRON_NAME]", <<~TXT
+    Deploy to environ identified by name.
+    Name can contain letters, numbers, dashes or underscores.
+    If environ name is left blank, current git branch name is used.
+  TXT
+  method_option :account_id, type: :numeric
+  method_option :region, type: :string
+  method_option :project, type: :string
   method_option :commit, type: :string, default: "HEAD"
   method_option :dry_run, type: :boolean, default: false
-  def deploy(environment)
-    abort_if_aws_cli_is_missing
-    Envirobly::Deployment.new environment, options
-  end
-
-  desc "set_access_token TOKEN", "Save and use an access token generated at Envirobly"
-  def set_access_token
-    token = ask("Access Token:", echo: false).strip
+  def deploy(environ_name = nil)
+    commit = Envirobly::Git::Commit.new options.commit
 
-    if token.blank?
-      $stderr.puts
-      $stderr.puts "Token can't be empty."
+    unless commit.exists?
+      say_error "Commit '#{commit.ref}' doesn't exist in this repository. Aborting."
       exit 1
     end
 
-    Envirobly::AccessToken.new(token).save
-  end
+    Envirobly::AccessToken.new(shell:).require!
 
-  private
-    def abort_if_aws_cli_is_missing
-      `which aws`
-      unless $?.success?
-        $stderr.puts "AWS CLI is missing. Please install it first: https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html"
-        exit 1
+    environ_name = environ_name.presence || commit.current_branch
+    project_name = nil
+    project_id = nil
+
+    if options.project.present?
+      if options.project =~ Envirobly::Defaults::Project.regexp
+        project_id = $1.to_i
+      else
+        project_name = options.project
       end
     end
+
+    deployment = Envirobly::Deployment.new(
+      account_id: options.account_id,
+      region: options.region,
+      project_name:,
+      environ_name:,
+      project_id:,
+      commit:,
+      shell:
+    )
+    deployment.perform(dry_run: options.dry_run)
+  end
+
+  desc "pull", "Download build context"
+  def pull(region, bucket, ref, path)
+    Envirobly::Duration.measure("Build context download took %s") do
+      s3 = Envirobly::Aws::S3.new(region:, bucket:)
+      s3.pull ref, path
+    end
+  end
 end

data/lib/envirobly/colorize.rb

@@ -0,0 +1,54 @@
+module Envirobly::Colorize
+  GREEN  = "\e[32m"
+  RED    = "\e[31m"
+  YELLOW = "\e[33m"
+  BLUE   = "\e[34m"
+  RESET  = "\e[0m"
+  BOLD   = "\e[1m"
+  FAINT  = "\e[2m"
+
+  def faint(text)
+    [ FAINT, text, RESET ].join
+  end
+
+  def bold(text)
+    [ BOLD, text, RESET ].join
+  end
+
+  def green(text)
+    [ GREEN, text, RESET ].join
+  end
+
+  def yellow(text)
+    [ YELLOW, text, RESET ].join
+  end
+
+  def red(text)
+    [ RED, text, RESET ].join
+  end
+
+  def green_check
+    green("✔")
+  end
+
+  def downwards_arrow_to_right
+    "↳"
+  end
+
+  def cross
+    "✖"
+  end
+
+  def display_config_errors(errors)
+    puts "#{red(cross)} Config contains the following issues:"
+
+    errors.each do |error|
+      puts
+      puts "  #{error["message"]}"
+
+      if error["path"]
+        puts faint("  #{downwards_arrow_to_right} #{error["path"]}")
+      end
+    end
+  end
+end