imagemaster3000 0.1.0

data/config/definitions/files/serial-getty@ttyS0.service

@@ -0,0 +1,35 @@
+#  This file is part of systemd.
+#
+#  systemd is free software; you can redistribute it and/or modify it
+#  under the terms of the GNU Lesser General Public License as published by
+#  the Free Software Foundation; either version 2.1 of the License, or
+#  (at your option) any later version.
+
+[Unit]
+Description=Serial Getty on %I
+Documentation=man:agetty(8) man:systemd-getty-generator(8)
+Documentation=http://0pointer.de/blog/projects/serial-console.html
+BindsTo=dev-%i.device
+After=dev-%i.device systemd-user-sessions.service plymouth-quit-wait.service
+After=rc-local.service
+
+# If additional gettys are spawned during boot then we should make
+# sure that this is synchronized before getty.target, even though
+# getty.target didn't actually pull it in.
+Before=getty.target
+IgnoreOnIsolate=yes
+
+[Service]
+ExecStart=-/sbin/agetty --autologin root --keep-baud 115200,38400,9600 %I $TERM
+Type=idle
+Restart=always
+UtmpIdentifier=%I
+TTYPath=/dev/%I
+TTYReset=yes
+TTYVHangup=yes
+KillMode=process
+IgnoreSIGPIPE=no
+SendSIGHUP=yes
+
+[Install]
+WantedBy=getty.target

data/config/definitions/files/ttyS0.conf

@@ -0,0 +1,22 @@
+# ttyS0 - getty
+#
+# This service maintains a getty on ttyS0 from the point the system is
+# started until it is shut down again.
+
+start on stopped rc RUNLEVEL=[2345] and (
+            not-container or
+            container CONTAINER=lxc or
+            container CONTAINER=lxc-libvirt)
+
+stop on runlevel [!2345]
+
+pre-start script
+    # getty will not be started if the serial console is not present
+    stty -F /dev/ttyS0 -a 2> /dev/null > /dev/null || { stop ; exit 0; }
+end script
+
+respawn
+script
+    exec /sbin/getty --autologin root -L ttyS0 115200 vt102
+end script
+# CLOUD_IMG: This file was created/modified by the Cloud Image build process

data/config/definitions/files/ubuntu-cloud.cfg

@@ -0,0 +1,111 @@
+# If this is set, 'root' will not be able to ssh in and they
+# will get a message to login instead as the above $user (ubuntu)
+disable_root: False
+user: root
+ssh_pwauth: False
+ssh_deletekeys: True
+ssh_genkeytypes: ['rsa', 'dsa']
+ssh_svcname: sshd
+
+# This will cause the set+update hostname module to not operate (if true)
+preserve_hostname: false
+cc_ready_cmd: ['/bin/true']
+mount_default_fields: [~, ~, 'auto', 'defaults,nofail', '0', '2']
+syslog_fix_perms: ~
+manage_etc_hosts: True
+
+# Update and upgrade system on first boot
+apt_preserve_sources_list: True
+package_update: True
+package_upgrade: True
+package_reboot_if_required: True
+
+# work only with OpenNebula, use network based datasource,
+# so that we can successfully resolve IPv4 based hostname
+disable_ec2_metadata: True
+datasource_list: ['OpenNebula']
+datasource:
+  OpenNebula:
+    dsmode: net
+
+# The modules that run in the 'init' stage
+cloud_init_modules:
+ - migrator
+ - seed_random
+ - bootcmd
+ - write-files
+ - growpart
+ - resizefs
+ - set_hostname
+ - update_hostname
+ - update_etc_hosts
+ - ca-certs
+ - rsyslog
+ - users-groups
+ - ssh
+
+# The modules that run in the 'config' stage
+cloud_config_modules:
+# Emit the cloud config ready event
+# this can be used by upstart jobs for 'start on cloud-config'.
+ - emit_upstart
+ - disk_setup
+ - mounts
+ - ssh-import-id
+ - locale
+ - set-passwords
+ - grub-dpkg
+ - apt-pipelining
+ - apt-configure
+ - package-update-upgrade-install
+ - landscape
+ - timezone
+ - puppet
+ - chef
+ - salt-minion
+ - mcollective
+ - disable-ec2-metadata
+ - runcmd
+ - byobu
+
+# The modules that run in the 'final' stage
+cloud_final_modules:
+ - rightscale_userdata
+ - scripts-per-once
+ - scripts-per-boot
+ - scripts-per-instance
+ - scripts-user
+ - ssh-authkey-fingerprints
+ - keys-to-console
+ - phone-home
+ - final-message
+ - power-state-change
+
+# System and/or distro specific settings
+# (not accessible to handlers/transforms)
+system_info:
+   # This will affect which distro class gets used
+   distro: ubuntu
+   # Other config here will be given to the distro class and/or path classes
+   paths:
+      cloud_dir: /var/lib/cloud/
+      templates_dir: /etc/cloud/templates/
+      upstart_dir: /etc/init/
+   package_mirrors:
+     - arches: [i386, amd64]
+       failsafe:
+         primary: http://archive.ubuntu.com/ubuntu
+         security: http://security.ubuntu.com/ubuntu
+       search:
+         primary:
+           - http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/
+           - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/
+           - http://%(region)s.clouds.archive.ubuntu.com/ubuntu/
+         security: []
+     - arches: [armhf, armel, default]
+       failsafe:
+         primary: http://ports.ubuntu.com/ubuntu-ports
+         security: http://ports.ubuntu.com/ubuntu-ports
+   ssh_svcname: ssh
+
+# vim:syntax=yaml

data/config/definitions/ubuntu-14.04.json

@@ -0,0 +1,30 @@
+{
+  "name":"METACLOUD-Ubuntu-14.04-x86_64@metacloud-dukan",
+  "url":"https://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img",
+  "verification":{
+    "signature":{
+      "detached":{
+        "signature":"https://cloud-images.ubuntu.com/trusty/current/SHA256SUMS.gpg",
+        "data":"https://cloud-images.ubuntu.com/trusty/current/SHA256SUMS"
+      }
+    },
+    "hash":{
+      "function":"SHA256"
+    }
+  },
+  "distribution":"Ubuntu",
+  "version":"14.04",
+  "actions":{
+    "copy":[
+      {
+        "source":"ubuntu-cloud.cfg",
+        "target":"/etc/cloud/",
+        "name":"cloud.cfg"
+      },
+      {
+        "source":"ttyS0.conf",
+        "target":"/etc/init/"
+      }
+    ]
+  }
+}

data/config/definitions/ubuntu-16.04.json

@@ -0,0 +1,30 @@
+{
+  "name":"METACLOUD-Ubuntu-16.04-x86_64@metacloud-dukan",
+  "url":"https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img",
+  "verification":{
+    "signature":{
+      "detached":{
+        "signature":"https://cloud-images.ubuntu.com/xenial/current/SHA256SUMS.gpg",
+        "data":"https://cloud-images.ubuntu.com/xenial/current/SHA256SUMS"
+      }
+    },
+    "hash":{
+      "function":"SHA256"
+    }
+  },
+  "distribution":"Ubuntu",
+  "version":"16.04",
+  "actions":{
+    "copy":[
+      {
+        "source":"ubuntu-cloud.cfg",
+        "target":"/etc/cloud/",
+        "name":"cloud.cfg"
+      },
+      {
+        "source":"serial-getty@ttyS0.service",
+        "target":"/etc/systemd/system/getty.target.wants/"
+      }
+    ]
+  }
+}

data/config/imagemaster3000.yml

@@ -0,0 +1,15 @@
+imagemaster3000:
+  definitions-dir: # If set, definitions in this direcotry are used to download and modify images
+  image-dir: /var/spool/imagemaster3000/images/ # Directory where to temporarily store images
+  image-list: /var/spool/imagemaster3000/image-list/imagemaster3000.list # Name and path of generated image list
+  endpoint: http://localhost/ # Endpoint where image list will be available
+  certificate: /etc/grid-security/cert.pem # Certificate to sign image list with
+  key: /etc/grid-security/key.pem # Key to sign image list with
+  group: imagemaster3000 # Group, images will be uploaded to
+  binaries:
+    virt-copy-in: /usr/bin/virt-copy-in # Path to binary needed for 'copy' action
+    guestfish: /usr/bin/guestfish # Path to binary needed for 'remove' action
+  logging:
+    level: ERROR # Logging level
+    file: /var/log/imagemaster3000/imagemaster3000.log # File to write log to. To turn off file logging leave this field empty.
+  debug: false # Debug mode

data/imagemaster3000.gemspec

@@ -0,0 +1,42 @@
+# coding: utf-8
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'imagemaster3000/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'imagemaster3000'
+  spec.version       = Imagemaster3000::VERSION
+  spec.authors       = ['Michal Kimle']
+  spec.email         = ['kimle.michal@gmail.com']
+
+  spec.summary       = 'Downloading and slight modification of cloud images'
+  spec.description   = 'Downloading and slight modification of cloud images'
+  spec.homepage      = 'https://github.com/Misenko/imagemaster3000'
+  spec.license       = 'Apache License, Version 2.0'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.15'
+  spec.add_development_dependency 'rake', '~> 12.0'
+  spec.add_development_dependency 'rspec', '~> 3.5'
+  spec.add_development_dependency 'rubocop', '~> 0.48'
+  spec.add_development_dependency 'rubocop-rspec', '~> 1.15'
+  spec.add_development_dependency 'simplecov', '~> 0.12'
+  spec.add_development_dependency 'pry', '~> 0.10'
+  spec.add_development_dependency 'vcr', '~> 3.0'
+  spec.add_development_dependency 'webmock', '~> 3.0'
+
+  spec.add_runtime_dependency 'thor', '~> 0.19'
+  spec.add_runtime_dependency 'yell', '~> 2.0'
+  spec.add_runtime_dependency 'settingslogic', '~> 2.0'
+  spec.add_runtime_dependency 'activesupport', '>= 4.0', '< 6.0'
+  spec.add_runtime_dependency 'tilt', '~> 2.0'
+  spec.add_runtime_dependency 'gpgme', '~> 2.0'
+  spec.add_runtime_dependency 'mixlib-shellout', '~> 2.2'
+  spec.add_runtime_dependency 'json-schema', '~> 2.8'
+
+  spec.required_ruby_version = '>= 2.2.0'
+end

data/lib/imagemaster3000/actions/copy.rb

@@ -0,0 +1,45 @@
+module Imagemaster3000
+  module Actions
+    class Copy
+      attr_accessor :filename, :source, :target, :name
+
+      def initialize(source: nil, target: nil, name: nil)
+        raise Imagemaster3000::Errors::ArgumentError, 'neither source nor target can be nil' if source.empty? || target.empty?
+
+        @filename = source
+        @source = File.join(Imagemaster3000::Settings[:'definitions-dir'], 'files', source)
+        @target = target
+        @name = name
+        logger.debug "Created action #{inspect}"
+      end
+
+      def run(image_file)
+        logger.debug "Running 'copy' action with source #{source.inspect} and target #{target.inspect} on file #{image_file.inspect}"
+        copy image_file
+        rename image_file if name
+      rescue Imagemaster3000::Errors::CommandExecutionError => ex
+        raise Imagemaster3000::Errors::ActionError, ex
+      end
+
+      private
+
+      def copy(image_file)
+        Imagemaster3000::Utils::CommandExecutioner.execute Imagemaster3000::Settings[:'binaries-virt-copy-in'],
+                                                           '-a',
+                                                           image_file,
+                                                           source,
+                                                           target
+      end
+
+      def rename(image_file)
+        Imagemaster3000::Utils::CommandExecutioner.execute Imagemaster3000::Settings[:'binaries-guestfish'],
+                                                           '-a',
+                                                           image_file,
+                                                           '-i',
+                                                           'mv',
+                                                           File.join(target, filename),
+                                                           File.join(target, name)
+      end
+    end
+  end
+end

data/lib/imagemaster3000/actions/remove.rb

@@ -0,0 +1,24 @@
+module Imagemaster3000
+  module Actions
+    class Remove
+      attr_accessor :file
+
+      def initialize(file)
+        @file = file
+        logger.debug "Created action #{inspect}"
+      end
+
+      def run(image_file)
+        logger.debug "Running 'remove' action with argument #{file.inspect} on file #{image_file.inspect}"
+        Imagemaster3000::Utils::CommandExecutioner.execute Imagemaster3000::Settings[:'binaries-guestfish'],
+                                                           '-a',
+                                                           image_file,
+                                                           '-i',
+                                                           'rm',
+                                                           file
+      rescue Imagemaster3000::Errors::CommandExecutionError => ex
+        raise Imagemaster3000::Errors::ActionError, ex
+      end
+    end
+  end
+end

data/lib/imagemaster3000/actions.rb

@@ -0,0 +1,6 @@
+module Imagemaster3000
+  module Actions
+    autoload :Copy, 'imagemaster3000/actions/copy'
+    autoload :Remove, 'imagemaster3000/actions/remove'
+  end
+end

data/lib/imagemaster3000/cli.rb

@@ -0,0 +1,115 @@
+require 'thor'
+require 'yell'
+
+module Imagemaster3000
+  class CLI < Thor
+    class_option :'logging-level',
+                 required: true,
+                 default: Imagemaster3000::Settings['logging']['level'],
+                 type: :string,
+                 enum: Yell::Severities
+    class_option :'logging-file',
+                 default: Imagemaster3000::Settings['logging']['file'],
+                 type: :string,
+                 desc: 'File to write logs to'
+    class_option :debug,
+                 default: Imagemaster3000::Settings['debug'],
+                 type: :boolean,
+                 desc: 'Runs in debug mode'
+
+    method_option :'definitions-dir',
+                  default: Imagemaster3000::Settings['definitions-dir'],
+                  type: :string,
+                  desc: 'If set, definitions in this direcotry are used to download and modify images'
+    method_option :'image-dir',
+                  default: Imagemaster3000::Settings['image-dir'],
+                  type: :string,
+                  required: true,
+                  desc: 'Directory where to temporarily store images'
+    method_option :group,
+                  required: true,
+                  default: Imagemaster3000::Settings['group'],
+                  type: :string,
+                  desc: 'Group, images will be uploaded to'
+    method_option :'image-list',
+                  required: true,
+                  default: Imagemaster3000::Settings['image-list'],
+                  type: :string,
+                  desc: 'Name and path of generated image list'
+    method_option :endpoint,
+                  required: true,
+                  default: Imagemaster3000::Settings['endpoint'],
+                  type: :string,
+                  desc: 'Endpoint where image list will be available'
+    method_option :certificate,
+                  required: true,
+                  default: Imagemaster3000::Settings['certificate'],
+                  type: :string,
+                  desc: 'Certificate to sign image list with'
+    method_option :key,
+                  required: true,
+                  default: Imagemaster3000::Settings['key'],
+                  type: :string,
+                  desc: 'Key to sign image list with'
+    method_option :'binaries-virt-copy-in',
+                  required: true,
+                  default: Imagemaster3000::Settings['binaries']['virt-copy-in'],
+                  type: :string,
+                  desc: 'Path to binary needed for \'copy\' action'
+    method_option :'binaries-guestfish',
+                  required: true,
+                  default: Imagemaster3000::Settings['binaries']['guestfish'],
+                  type: :string,
+                  desc: 'Path to binary needed for \'remove\' action'
+
+    desc 'start', 'Downloads images and generates image list'
+    def start
+      initialize_configuration options
+      initialize_logger
+      logger.debug "Imagemaster3000 'start' called with parameters: #{Imagemaster3000::Settings.to_hash.inspect}"
+
+      Imagemaster3000::MainProcess.new.run
+    end
+
+    desc 'version', 'Prints imagemaster3000 version'
+    def version
+      $stdout.puts Imagemaster3000::VERSION
+    end
+
+    default_task :start
+
+    private
+
+    def initialize_configuration(options)
+      Imagemaster3000::Settings.clear
+      Imagemaster3000::Settings.merge! options.to_hash
+
+      gem_dir = File.realdirpath(File.join(File.dirname(__FILE__), '..', '..'))
+      Imagemaster3000::Settings[:'definitions-dir'] = File.join(gem_dir, 'config', 'definitions') \
+        unless Imagemaster3000::Settings[:'definitions-dir']
+    end
+
+    def initialize_logger
+      Imagemaster3000::Settings[:'logging-level'] = 'DEBUG' if Imagemaster3000::Settings[:debug]
+
+      logging_file = Imagemaster3000::Settings[:'logging-file']
+      logging_level = Imagemaster3000::Settings[:'logging-level']
+
+      Yell.new :stdout, name: Object, level: logging_level.downcase, format: Yell::DefaultFormat
+      Object.send :include, Yell::Loggable
+
+      setup_file_logger(logging_file) if logging_file
+
+      logger.debug 'Running in debug mode...'
+    end
+
+    def setup_file_logger(logging_file)
+      unless (File.exist?(logging_file) && File.writable?(logging_file)) || File.writable?(File.dirname(logging_file))
+        logger.error "File #{logging_file} isn't writable"
+        return
+      end
+
+      logger.adapter :file, logging_file
+    end
+  end
+end