aethernal-agent 0.2.0 → 0.3.6

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    aethernal-agent (0.2.0)
+    aethernal-agent (0.3.6)
       activesupport
-      httparty
+      bcrypt
+      docker-api (~> 1.34.2)
       sinatra (~> 2.0.7)
       sinatra-contrib
       sqlite3
@@ -11,38 +12,36 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.3)
+    activesupport (6.0.3.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
       zeitwerk (~> 2.2, >= 2.2.2)
-    backports (3.17.1)
-    coderay (1.1.2)
-    concurrent-ruby (1.1.6)
-    diff-lcs (1.3)
-    httparty (0.18.0)
-      mime-types (~> 3.0)
-      multi_xml (>= 0.5.2)
-    i18n (1.8.2)
+    backports (3.18.2)
+    bcrypt (3.1.16)
+    coderay (1.1.3)
+    concurrent-ruby (1.1.7)
+    diff-lcs (1.4.4)
+    docker-api (1.34.2)
+      excon (>= 0.47.0)
+      multi_json
+    excon (0.76.0)
+    i18n (1.8.5)
       concurrent-ruby (~> 1.0)
-    method_source (0.9.2)
-    mime-types (3.3.1)
-      mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.0512)
-    minitest (5.14.0)
-    multi_json (1.14.1)
-    multi_xml (0.6.0)
+    method_source (1.0.0)
+    minitest (5.14.2)
+    multi_json (1.15.0)
     mustermann (1.1.1)
       ruby2_keywords (~> 0.0.1)
-    net-scp (2.0.0)
-      net-ssh (>= 2.6.5, < 6.0.0)
-    net-ssh (5.2.0)
+    net-scp (3.0.0)
+      net-ssh (>= 2.6.5, < 7.0.0)
+    net-ssh (6.1.0)
     net-telnet (0.1.1)
-    pry (0.12.2)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    rack (2.2.2)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rack (2.2.3)
     rack-protection (2.0.8.1)
       rack
     rake (10.5.0)
@@ -50,18 +49,18 @@ GEM
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
       rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.0)
-      rspec-support (~> 3.9.0)
-    rspec-expectations (3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-its (1.3.0)
       rspec-core (>= 3.0.0)
       rspec-expectations (>= 3.0.0)
-    rspec-mocks (3.9.0)
+    rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
-    rspec-support (3.9.0)
+    rspec-support (3.9.3)
     ruby2_keywords (0.0.2)
     serverspec (2.41.5)
       multi_json
@@ -81,7 +80,7 @@ GEM
       rack-protection (= 2.0.8.1)
       sinatra (= 2.0.8.1)
       tilt (~> 2.0)
-    specinfra (2.82.4)
+    specinfra (2.82.19)
       net-scp
       net-ssh (>= 2.7)
       net-telnet (= 0.1.1)
@@ -91,7 +90,7 @@ GEM
     tilt (2.0.10)
     tzinfo (1.2.7)
       thread_safe (~> 0.1)
-    zeitwerk (2.3.0)
+    zeitwerk (2.4.0)
 
 PLATFORMS
   ruby

data/README.md

@@ -1,35 +1,6 @@
 # AethernalAgent
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/aethernal_agent`. To experiment with that code, run `bin/console` for an interactive prompt.
+This is pre-production code, if you like to work on this please reach
+out.
 
-TODO: Delete this and the text above, and describe your gem
-
-## Installation
-
-Add this line to your application's Gemfile:
-
-```ruby
-gem 'aethernal_agent'
-```
-
-And then execute:
-
-    $ bundle
-
-Or install it yourself as:
-
-    $ gem install aethernal_agent
-
-## Usage
-
-TODO: Write usage instructions here
-
-## Development
-
-After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
-
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
-
-## Contributing
-
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/aethernal_agent.
+This file will be updated as soon as we reach Alpha state.

data/Vagrantfile

@@ -10,6 +10,7 @@ Vagrant.configure(2) do |config|
 
   config.vm.provider "virtualbox" do |vb|
     vb.memory = "2048"
+    vb.cpus = 4
     disk_file = File.expand_path("../.vagrant/disks/c-disk1.vdi", __FILE__)
 
     if ! File.exist?(disk_file)
@@ -19,12 +20,19 @@ Vagrant.configure(2) do |config|
   end
 
   config.vm.provision :shell, privileged: true, inline: <<-SHELL
+       apt update
+       apt install -y build-essential libsqlite3-dev snapd apt-transport-https ca-certificates curl gnupg-agent software-properties-common
        snap install ruby --classic
+       curl -fsSL https://download.docker.com/linux/ubuntu/gpg > gpg
+       apt-key add gpg
+       sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+       apt install -y docker-ce docker-ce-cli containerd.io libssl-dev
        adduser --add_extra_groups --gecos '' --quiet --disabled-password testuser
-       apt update
-       apt install -y build-essential libsqlite3-dev
+       usermod -a -G docker testuser
   SHELL
 
+  config.vm.provision :reload
+
   config.trigger.after [:up, :reload] do |trigger|
     trigger.info = "Doing bundle install"
     trigger.run_remote = {inline: "cd /vagrant && /snap/bin/bundle install --system", privileged: true}

data/aethernal-agent.gemspec

@@ -41,5 +41,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency "sinatra", "~> 2.0.7"
   spec.add_dependency "sinatra-contrib"
   spec.add_dependency "sqlite3"
-  spec.add_dependency "httparty"
+  spec.add_dependency "bcrypt"
+  spec.add_dependency "docker-api", "~> 1.34.2"
 end

data/bin/install-aa

@@ -0,0 +1,120 @@
+#!/bin/bash
+
+is_root() {
+  [[ $(id -u) -eq 0 ]]
+}
+
+is_ubuntu() {
+  grep "Ubuntu" "/etc/os-release" &>/dev/null
+}
+
+run() {
+  local cmd=$@
+  info "running \"${cmd}\""
+  $cmd
+
+  local status=$?
+  if [[ $status -ne 0 ]]; then
+    fail "failed to run \"${cmd}\", exit status ${status}"
+  fi
+}
+timestamp() {
+  date "+%H:%M:%S.%3N"
+}
+
+info() {
+  local msg=$1
+  echo -e "\e[1;32m===> $(timestamp) ${msg}\e[0m"
+}
+
+warn() {
+  local msg=$1
+  echo -e "\e[1;33m===> $(timestamp) ${msg}\e[0m"
+}
+
+fail() {
+  local msg=$1
+  echo -e "\e[1;31m===> $(timestamp) ERROR: ${msg}\e[0m"
+  exit 1
+}
+
+aethernal_exists(){
+  [ -e "" ]
+}
+start_aa(){
+  run service aethernal-agent enable
+  run service aethernal-agent start
+}
+stop_aa(){
+  run service aethernal-agent stop
+}
+restart_aa(){
+  run service aethernal-agent restart
+}
+update_services(){
+      if ! curl -o /lib/systemd/system/aethernal-agent.service https://storage.googleapis.com/bysh-chef-files/aethernal-agent.service;then
+        fail "Failed to download systemd service file"
+      fi
+
+      run systemctl daemon-reload
+}
+
+install_deps(){
+  info "Ensuring dependencies are installed"
+  run apt update
+  run apt install -y apache2 unzip wget curl ruby2.5 build-essential libsqlite3-dev sqlite3 ruby-dev apt-transport-https ca-certificates curl software-properties-common
+}
+
+install_gem(){
+    info "Downloading latest Aethernal Agent gem release"
+    run gem install --no-doc aethernal-agent
+    restart_aa
+}
+
+install_ci(){
+  info "Downloading latest Aethernal Agent release from CI build"
+   run gem install bundler --no-document
+   if ! wget https://gitlab.com/bytesized/aethernal-agent/-/jobs/artifacts/master/download?job=build -O /tmp/aa.zip;then
+     fail "Failed to download gem from CI"
+   fi
+
+   run unzip -o /tmp/aa.zip -d /tmp/
+   run gem install --no-document /tmp/pkg/aethernal-agent*.gem
+   restart_aa
+}
+
+main() {
+  if ! is_root; then
+    fail "this script must be executed as the root user"
+  fi
+
+  if ! is_ubuntu; then
+    fail "this script is only compatible with Ubuntu"
+  fi
+
+  local ubuntu_version=$(cat /etc/os-release | grep VERSION_ID| cut -d = -f2 | sed "s/\"//g")
+  local version_name=$(lsb_release -a | grep "Codename" | awk '{print $2}')
+
+  if [[ "${version_name}" = "" ]];then
+    fail "Sorry but an unsupported Ubuntu version was found $ubuntu_version - $version_name"
+  fi
+  info "Found Ubuntu version ${ubuntu_version}"
+
+  install_deps
+  PKG_OK=$(dpkg-query -W --showformat='${Status}\n' docker-ce|grep "install ok installed")
+  if [ "" = "$PKG_OK" ]; then
+    echo "Docker not installed, doing that now"
+    run curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+    add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+    run apt-get install -y docker-ce
+  fi
+  update_services
+
+  if [[ "$1" = "from-ci" ]];then
+    install_ci
+  else
+    install_gem
+  fi
+}
+
+main $@

data/bin/run-aa-for-test

@@ -26,7 +26,7 @@ unless app && method && user && options
   return
 end
 options.reverse_merge!(user: user)
-plugin = "AethernalAgent::#{app.titlecase}".constantize.new(options)
+plugin = "AethernalAgent::#{app.titlecase.gsub(' ','')}".constantize.new(options)
 
 puts "Running BOSS #{AethernalAgent::VERSION} - AethernalAgent::#{app.titlecase}.#{method}(#{options})"
 result = plugin.send(method, options)

data/lib/aethernal_agent.rb

@@ -1,4 +1,5 @@
 require 'yaml'
+require 'docker-api'
 require 'sqlite3'
 require 'active_support/core_ext/hash/indifferent_access'
 require 'active_support/core_ext/string'
@@ -7,9 +8,15 @@ require 'etc'
 require 'fileutils'
 require 'socket'
 require 'logger'
+require 'bcrypt'
+require 'net/http'
 
 require 'aethernal_agent/app'
 require 'aethernal_agent/loader'
+require 'aethernal_agent/docker/volume'
+require 'aethernal_agent/docker/port_binding'
+require 'aethernal_agent/docker/container_settings'
+require 'aethernal_agent/docker/env'
 require 'aethernal_agent/template'
 require 'aethernal_agent/operation'
 require 'aethernal_agent/operation_pool'

data/lib/aethernal_agent/apache/apache.rb

@@ -6,16 +6,19 @@ module AethernalAgent
     include AethernalAgent::Filesystem
     include AethernalAgent::Utils
     include AethernalAgent::Errors
+    extend AethernalAgent::Utils
 
     attr_accessor :user
 
-    def initialize(options = {})
-      self.user = options['user'] || 'root'
+    def self.ensure_default_config
+      run_command("a2enmod rewrite proxy scgi headers proxy proxy_html proxy_http ssl xml2enc proxy_wstunnel mpm_itk")
+      run_command("echo 'ProxyPass /aethernal http://127.0.0.1:4567/' > /etc/apache2/sites-available/aa-default.conf")
+      run_command("a2ensite aa-default")
+      run_command("systemctl restart apache2")
     end
 
-    def ensure_mods
-      run_command("a2enmod proxy scgi headers proxy proxy_html proxy_http ssl xml2enc proxy_wstunnel")
-      self.restart
+    def initialize(options = {})
+      self.user = options['user'] || 'root'
     end
 
     def write_app_config(template_path, app_name, port)

data/lib/aethernal_agent/apache/templates/aa-default.conf

@@ -0,0 +1,2 @@
+# Default pass should show up on all routes for maximum compatibility
+ProxyPass /aethernal http://127.0.0.1:4567/

data/lib/aethernal_agent/apache/templates/aa-user.conf.erb

@@ -9,6 +9,7 @@
   <% end %>
 
   DocumentRoot <%= @home_folder %>/www
+  AssignUserId <%= @user %> aethernal
   CustomLog /var/log/apache2/<%= @user %>.base-access.log combined
   ErrorLog /var/log/apache2/<%= @user %>.base-error.log
   IncludeOptional users/<%= @user %>/*.conf
@@ -18,4 +19,5 @@
     Require all granted
   </Directory>
 
+  ProxyPass /aethernal http://127.0.0.1:4567/
 </VirtualHost>

data/lib/aethernal_agent/app.rb

@@ -17,7 +17,7 @@ module AethernalAgent
     include AethernalAgent::Utils
     include AethernalAgent::Errors
 
-    attr_accessor :manifest, :user, :plugin_path, :global_options
+    attr_accessor :manifest, :user, :plugin_path, :global_options, :container_settings
 
     def initialize(options = {})
       options = HashWithIndifferentAccess.new(options)
@@ -85,6 +85,13 @@ module AethernalAgent
           elsif opts["auto_extract"]
             extract(opts["target_name"], delete_after: true)
           end
+        when 'git'
+          run_command("mkdir #{home_folder_path(File.join('apps'))}")
+          run_command("mkdir #{app_path}")
+          run_command("git clone #{self.manifest['package']['git']['url']} #{File.join(app_path)}")
+        when 'docker'
+          options = self.manifest.package['docker']
+          run_command("docker pull #{options['image']}")
         end
       end
       AethernalAgent.logger.info("Done Installing packages for #{self.manifest.name} - v#{self.manifest.version}")
@@ -100,10 +107,13 @@ module AethernalAgent
       self.manifest.package.each_key do |package_type|
         case package_type
         when "apt"
-          AethernalAgent.logger.debug("Using apt for ubuntu #{ubuntu_release}")
-          packages = self.manifest.package[package_type][ubuntu_release]
-          AethernalAgent.logger.debug("Removing packages #{packages}")
-          apt_package(packages: packages, action: :remove) if packages
+          options = self.manifest.package['apt'][ubuntu_release]
+          AethernalAgent.logger.debug("Using apt for ubuntu #{ubuntu_release} - #{options}")
+          packages = options['packages']
+          if packages
+            AethernalAgent.logger.debug("Removing packages #{packages}")
+            apt_package(packages: packages, action: :remove)
+          end
         when "direct_download"
           directory(app_path, action: :delete)
         end
@@ -123,29 +133,65 @@ module AethernalAgent
         apache.write_app_config(template_path(self.manifest.apache_configuration), self.manifest.plain_name, opts[:port])
       end
 
+      if self.is_docker_install?
+        begin
+          docker_group = Etc.getgrnam('docker')
+        rescue ArgumentError
+          self.add_errors("Docker group not found, please install Docker (https://docs.docker.com/engine/install/ubuntu/). If Docker is already installed make sure the docker group exists.") and return create_return_args(opts)
+        end
+
+        unless docker_group.mem.include?(self.user)
+          self.add_errors("#{self.user} is not a member of the Docker group. Because of the implied security implications aethernal-agent does not do this by default. Please use `usermod -aG docker #{self.user}` if you want to give this user Docker access and restart your Capsule so SystemD has access as well.") and return create_return_args(opts)
+        end
+        self.container_settings = AethernalAgent::Docker::ContainerSettings.build_from_options(self.manifest["actions"]["configure_app_user"], opts)
+      end
+
       AethernalAgent.logger.debug("Running custom plugin code.")
       yield opts if block_given?
       AethernalAgent.logger.debug("Done running custom plugin code.")
 
+      # This is probably not the best way to check if we need a docker image
+      if self.is_docker_install?
+        image = self.manifest.package['docker']['image']
+        AethernalAgent.logger.debug("Docker image information found")
+
+        # Pull the image
+        AethernalAgent.logger.debug("Pulling image #{image}")
+        ::Docker::Image.create('fromImage' => image)
+
+        begin
+          container = self.container_settings.create_container(self.docker_container_name, image)
+          AethernalAgent.logger.debug("Starting: #{container.start}")
+        rescue StandardError => e
+          self.add_errors(e)
+          return create_return_args(opts)
+        end
+      end
+
       AethernalAgent.logger.debug("Starting service generation.")
-      self.manifest.services.each do |s|
-        options = {service_name: "#{s}.service", user: self.user, vars: opts}
-        service_template = template_path("#{s}.service.erb")
-        if File.exists?(service_template)
-          AethernalAgent.logger.debug("Found service template")
-          options[:template] = "#{s}.service.erb"
-        elsif File.exists?(files_path("#{s}.service"))
-          AethernalAgent.logger.debug("Found service file")
-          options[:file] = "#{s}.service"
+      if self.manifest.services.present?
+        self.manifest.services.each do |s|
+          options = {service_name: "#{s}.service", user: self.user, vars: opts}
+          service_template = template_path("#{s}.service.erb")
+          if File.exists?(service_template)
+            AethernalAgent.logger.debug("Found service template: '#{service_template}'")
+            options[:template] = "#{s}.service.erb"
+          elsif File.exists?(files_path("#{s}.service"))
+            AethernalAgent.logger.debug("Found service file")
+            options[:file] = "#{s}.service"
+          end
+          create_service_file(options)
         end
 
-        create_service_file(options)
+        AethernalAgent.logger.debug("Done with service generation.")
+      else
+        AethernalAgent.logger.debug("Skipping service generation, no service file defined.")
       end
-      AethernalAgent.logger.debug("Done with service generation.")
 
+      AethernalAgent.logger.debug("Done with service generation.")
       directory(home_folder_path("apps"), owner: self.user)
-
       AethernalAgent.logger.debug("Done running configure_app_user")
+
       return create_return_args(opts)
     end
 
@@ -155,8 +201,10 @@ module AethernalAgent
 
       run_user_systemctl(:stop)
 
-      self.manifest.services.each do |service|
-        file(service_file_path("#{service}.service"), action: :delete)
+      if self.manifest.services.present?
+        self.manifest.services.each do |service|
+          file(service_file_path("#{service}.service"), action: :delete)
+        end
       end
 
       if self.manifest.apache_configuration
@@ -164,8 +212,17 @@ module AethernalAgent
         apache.remove_app_config(self.manifest.plain_name)
       end
 
-      if self.is_local_install?
-        self.uninstall_packages(opts)
+      # Jackett is removing necessary packages, for now dont uninstall a package ever in user_configure
+      #if self.is_local_install?
+      #  self.uninstall_packages(opts)
+      #end
+      if self.is_docker_install?
+        begin
+          container = ::Docker::Container.get(self.docker_container_name)
+          container.delete(:force => true)
+        rescue ::Docker::Error::NotFoundError
+          AethernalAgent.logger.warn("Wanted to delete Docker container '#{self.docker_container_name}' but it did not exist.")
+        end
       end
 
       yield opts if block_given?
@@ -223,7 +280,7 @@ module AethernalAgent
           end
 
           # This option should be an autogenerated string (passwords)
-          if reqs.include?("auto_generate") && reqs["auto_generate"] == true && (!options.has_key?(opt) || options[opt].empty?)
+          if reqs.include?("auto_generate") && reqs["auto_generate"] == true && (!options.has_key?(opt) || options[opt].blank?)
             options[opt] = self.random_string
           end
 
@@ -289,6 +346,10 @@ module AethernalAgent
       case options[:extract_as].gsub('.', '')
       when "gz", "tar"
         run_command("tar -C #{extract_to} -xvf #{extract_from}")
+      when "tgz"
+        run_command("tar -C #{extract_to} -zxvf #{extract_from}")
+      when "bz2", "tar"
+        run_command("tar -C #{extract_to} -xvjf #{extract_from}")
       when "zip"
         run_command("unzip -od #{extract_to} #{extract_from}")
       else
@@ -306,6 +367,10 @@ module AethernalAgent
       self.manifest.installation_type == 'local_install'
     end
 
+    def is_docker_install?
+      self.manifest.installation_type == 'docker_image'
+    end
+
     def sha_1_hash(password, salt)
       sha = Digest::SHA1.new
       sha.update(salt)