envirobly-orchestra 0.4.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 27540543de48daec3396d69ad5ce05079fefd25a0f76a092c914a8e2873a80e7
+  data.tar.gz: 96d0c5e27e028b8c0cc06df90c81bc111a5efe4d591c1926d201773443bc9e11
+SHA512:
+  metadata.gz: 803c22a7247aa5e69a133436924146cb94d2d7d7f56258e446154fac304fcfdba4d4b60f84a1b9407af1505d12eb40946bb515ce19f1a6354b6ae73f178fc82a
+  data.tar.gz: d16b2d572fbb2acd1357c733af6083ea5020cc82eee376c771ffcab3ec5a795bccd9cb948f558247fcec5c27e02950a44add0d7247d0f497da8bb1f856d6bb55

data/LICENSE

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) 2023 Robert Starsi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice (including the next paragraph) shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/bin/orchestra

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require "orchestra"
+require "securerandom"
+
+Thread.current[:invocation] = SecureRandom.hex 4
+
+Orchestra::Cli::Main.start(ARGV)

data/lib/orchestra/base.rb

@@ -0,0 +1,44 @@
+require "thor"
+
+class Orchestra::Base < Thor
+  def self.exit_on_failure?
+    true
+  end
+
+  private
+    LOCKFILE_PATH       = "/tmp/orchestra.lock"
+    LOCK_RETRY_INTERVAL = 2
+    def with_lock
+      File.open(LOCKFILE_PATH, File::CREAT | File::EXCL | File::WRONLY) do
+        yield
+      end
+    rescue Errno::EEXIST
+      log "Execution lock in place, retrying in #{LOCK_RETRY_INTERVAL}s"
+
+      sleep LOCK_RETRY_INTERVAL
+
+      retry
+    ensure
+      if File.exist? LOCKFILE_PATH
+        File.delete LOCKFILE_PATH
+
+        log "lock released"
+      end
+    end
+
+    def log(text)
+      Orchestra::Logger.instance.log "inv-#{Thread.current[:invocation]} #{text}"
+    end
+
+    def execute(cmd, exit_on_failure: true)
+      stdout_and_err, status = Open3.capture2e *cmd
+
+      log stdout_and_err unless stdout_and_err.blank?
+
+      if exit_on_failure && status.to_i > 0
+        exit 1
+      end
+
+      status.to_i == 0
+    end
+end

data/lib/orchestra/cli/images.rb

@@ -0,0 +1,59 @@
+require "open3"
+require "pathname"
+require "benchmark"
+
+class Orchestra::Cli::Images < Orchestra::Base
+  desc "build", "Build and push a Docker image"
+  method_option :git_url,         type: :string, required: true
+  method_option :commit_id,       type: :string, required: true
+  method_option :cache_bucket,    type: :string, required: true
+  method_option :cache_region,    type: :string, required: true
+  method_option :image_uri,       type: :string, required: true
+  method_option :dockerfile,      type: :string, required: true
+  method_option :build_context,   type: :string, required: true
+  def build
+    status = 1
+
+    puts "Checking out commit #{options.commit_id}..."
+
+    checkout_time = Benchmark.realtime do
+      output, status =
+        Open3.capture2e "envirobly-git-checkout-commit", git_checkout_path.to_s, options.git_url, options.commit_id
+
+      puts output
+    end
+
+    puts "Checkout finished in #{checkout_time.to_i}s\n"
+
+    $stdout.flush
+
+    exit 1 if status.to_i > 0
+
+    exec *buildx_build_cmd
+  end
+
+  private
+    def buildx_build_cmd
+      [
+        "docker", "buildx", "build",
+        "--progress=plain",
+        "--cache-from=type=s3,region=#{options.cache_region},bucket=#{options.cache_bucket},name=app",
+        "--cache-to=type=s3,region=#{options.cache_region},bucket=#{options.cache_bucket},name=app,mode=max",
+        "-t", options.image_uri, "--push",
+        "-f", dockerfile_path.to_s,
+        build_context.to_s
+      ]
+    end
+
+    def git_checkout_path
+      Pathname.new "/tmp/build"
+    end
+
+    def build_context
+      git_checkout_path.join options.build_context
+    end
+
+    def dockerfile_path
+      git_checkout_path.join options.dockerfile
+    end
+end

data/lib/orchestra/cli/main.rb

@@ -0,0 +1,20 @@
+class Orchestra::Cli::Main < Orchestra::Base
+  desc "version", "Show Orchestra version"
+  def version
+    puts Orchestra::VERSION
+  end
+
+  desc "docker-version", "Show Docker version"
+  def docker_version
+    puts `docker -v`
+  end
+
+  desc "services", "Manage services"
+  subcommand "services", Orchestra::Cli::Services
+
+  desc "images", "Docker images"
+  subcommand "images", Orchestra::Cli::Images
+
+  desc "stack", "Manage stack"
+  subcommand "stack", Orchestra::Cli::Stack
+end

data/lib/orchestra/cli/services.rb

@@ -0,0 +1,171 @@
+require "open3"
+# require "httpx"
+require "pathname"
+
+class Orchestra::Cli::Services < Orchestra::Base
+  desc "up", "Start services based on config synced from config bucket"
+  method_option :config_dir,    type: :string, required: true
+  method_option :config_bucket, type: :string, required: true
+  method_option :config_region, type: :string, required: true
+  def up
+    with_lock do
+      log "services up"
+
+      execute sync_config_files_cmd
+      execute compose_prepare_cmd
+      initialize_data_volumes
+      execute compose_up_cmd
+    end
+  end
+
+  desc "down", "Stop services defined in supplied compose file"
+  method_option :config_dir, type: :string, required: true
+  def down
+    with_lock do
+      log "services down"
+
+      execute compose_down_cmd
+    end
+  end
+
+  desc "lock_test", "Test locking"
+  def lock_test
+    with_lock do
+      log "Executing something with lock for 15 seconds..."
+      sleep 15
+      log "Lock test done."
+    end
+  end
+
+  private
+    # def list_containers
+    #   http = HTTPX.with(transport: "unix", addresses: ["/var/run/docker.sock"])
+
+    #   response = http.get("http://localhost/v1.43/containers/json")
+
+    #   response.body
+    # end
+
+    # https://docs.docker.com/engine/reference/commandline/compose_up/
+    def compose_up_cmd
+      [
+        "docker", "compose",
+        "-f", compose_file_path,
+        "up",
+        "--quiet-pull",
+        "--remove-orphans",
+        "--detach",
+        "--wait"
+      ]
+    end
+
+    def compose_prepare_cmd
+      [
+        "docker", "compose",
+        "-f", compose_file_path,
+        "up",
+        "--quiet-pull",
+        "--detach",
+        "--no-start"
+      ]
+    end
+
+    def compose_down_cmd
+      [
+        "docker", "compose",
+        "-f", compose_file_path,
+        "down",
+        "--remove-orphans"
+      ]
+    end
+
+    def compose_file_path
+      Pathname.new(options.config_dir).join("compose.yml").to_s
+    end
+
+    def sync_config_files_cmd
+      [
+        "aws", "s3",
+        "cp", "--recursive", "--no-progress",
+        "--region", options.config_region,
+        "s3://#{options.config_bucket}",
+        options.config_dir
+      ]
+    end
+
+    def compose_definition
+      @compose_definition ||= YAML.load ERB.new(File.read(compose_file_path)).result, aliases: true
+    end
+
+    def initialize_data_volumes
+      compose_definition["services"].each do |_, service|
+        next if service.dig("labels", "envirobly.data-volume.pool").blank?
+
+        zpool              = service["labels"]["envirobly.data-volume.pool"]
+        mountpoint         = service["labels"]["envirobly.data-volume.pool.mountpoint"]
+        zfs_dataset        = service["labels"]["envirobly.data-volume.dataset"]
+        dataset_mountpoint = service["labels"]["envirobly.data-volume.dataset.mountpoint"]
+        device             = service["labels"]["envirobly.data-volume.device"]
+
+        if zfs_dataset_exist?(zfs_dataset)
+          log " ZFS dataset exists"
+          next
+        end
+
+        log "Looking for block device '#{device}'..."
+
+        wait_for_block_device device
+
+        log "Block device '#{device}' found"
+
+        if execute zpool_import_cmd(zpool), exit_on_failure: false
+          log "ZFS pool '#{zpool}' successfully imported"
+        else
+          execute zpool_create_cmd(zpool, device, mountpoint:)
+          execute set_zpool_compression_cmd(zpool)
+          execute create_zfs_dataset_cmd(zfs_dataset)
+          chmod_zfs_dataset_mountpoint(dataset_mountpoint)
+
+          log "Created new ZFS pool and dataset '#{zfs_dataset}'"
+        end
+      end
+    end
+
+    def zfs_dataset_exist?(dataset)
+      execute ["zfs", "list", dataset], exit_on_failure: false
+    end
+
+    def wait_for_block_device(device)
+      counter = 1
+
+      while !File.blockdev?(device) do
+        if counter > 15
+          log "Block device '#{device}' not found within 15s. Aborting"
+          exit 1
+        end
+
+        counter += 1
+        sleep 1
+      end
+    end
+
+    def zpool_import_cmd(pool)
+      ["zpool", "import", pool]
+    end
+
+    def zpool_create_cmd(pool, device, mountpoint:)
+      ["zpool", "create", "-m", mountpoint, pool, device]
+    end
+
+    def set_zpool_compression_cmd(pool)
+      ["zfs", "set", "compression=lz4", pool]
+    end
+
+    def create_zfs_dataset_cmd(dataset)
+      ["zfs", "create", dataset]
+    end
+
+    def chmod_zfs_dataset_mountpoint(path)
+      FileUtils.chmod 0777, path
+    end
+end

data/lib/orchestra/cli/stack.rb

@@ -0,0 +1,6 @@
+class Orchestra::Cli::Stack < Orchestra::Base
+  desc "prune", "Stop inactive services and clean up their data volumes"
+  def prune
+    # docker compose up -d --quiet-pull --no-recreate --remove-orphans --no-start
+  end
+end

data/lib/orchestra/cli.rb

@@ -0,0 +1 @@
+module Orchestra::Cli; end

data/lib/orchestra/logger.rb

@@ -0,0 +1,28 @@
+require "singleton"
+
+class Orchestra::Logger
+  include Singleton
+
+  LOG_FILE_PATH    = "/var/log/orchestra.log"
+  TIMESTAMP_FORMAT = "%Y%m%dT%H%M%SZ"
+
+  def initialize
+    @log_file = File.open(LOG_FILE_PATH, "a")
+  end
+
+  def log(text)
+    timestamp = Time.now.strftime(TIMESTAMP_FORMAT)
+    log_message = "#{timestamp} #{text}"
+
+    # Append to the log file
+    @log_file.puts(log_message)
+    @log_file.flush
+
+    # Print to stdout
+    puts log_message
+  end
+
+  def close_log_file
+    @log_file.close
+  end
+end

data/lib/orchestra/version.rb

@@ -0,0 +1,3 @@
+module Orchestra
+  VERSION = "0.4.1"
+end

data/lib/orchestra.rb

@@ -0,0 +1,10 @@
+module Orchestra
+end
+
+require "active_support"
+require "active_support/core_ext"
+require "zeitwerk"
+
+loader = Zeitwerk::Loader.for_gem
+loader.setup
+loader.eager_load

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: envirobly-orchestra
+version: !ruby/object:Gem::Version
+  version: 0.4.1
+platform: ruby
+authors:
+- Robert Starsi
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2024-01-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: debug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+description:
+email: klevo@klevo.sk
+executables:
+- orchestra
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- bin/orchestra
+- lib/orchestra.rb
+- lib/orchestra/base.rb
+- lib/orchestra/cli.rb
+- lib/orchestra/cli/images.rb
+- lib/orchestra/cli/main.rb
+- lib/orchestra/cli/services.rb
+- lib/orchestra/cli/stack.rb
+- lib/orchestra/logger.rb
+- lib/orchestra/version.rb
+homepage: https://klevo.sk
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.4
+signing_key:
+specification_version: 4
+summary: Container orchestration agent.
+test_files: []