pi_build_modifier 0.4.0.pre.alpha → 0.5.0.pre.alpha

data/pi_build_modifier/lib/pi_build_modifier/modify/modifiers.rb

@@ -0,0 +1,281 @@
+# Copyright (c) 2017-2019 Beate Ottenwälder
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require 'json'
+require 'erb'
+require 'fileutils'
+require 'pi_build_modifier/modify/configs'
+require 'pi_build_modifier/utils/logex'
+
+module PiBuildModifier
+
+  module Modifiers
+
+    ##
+    # ModifiersBuilder creates a Modifiers object and adds build configuration modifiers which map the (json) config to
+    # the the individual pi-gen build sources
+
+    class ModifiersBuilder
+      class << self
+
+        ##
+        # Initialization and configuration of the modifiers that are used to change and configure the build sources
+
+        def build_defaults(workspace, config)
+          modifiers = Modifiers.new
+          modifiers.with_json_configuration(config)
+          create_modifiers(workspace).each do |*, modifier|
+            modifiers.with_config_modifier(modifier)
+          end
+          modifiers
+        end
+
+        private def create_modifiers(workspace)
+          {
+              :erb_config => create_erb_modifier(workspace),
+              :config_file => create_config_file_modifier(workspace),
+              :default => create_config_modifier(workspace)
+          }
+        end
+
+        private def create_config_modifier(workspace)
+          config_modifier = ConfigModifier.new(workspace)
+          Configs::create_config_modifiers.each do |modifier|
+            config_modifier.add(modifier)
+          end
+          config_modifier
+        end
+
+        private def create_erb_modifier(workspace)
+          erb_modifier = ERBConfigModifier.new(workspace)
+          Configs::create_erb_modifiers.each do |modifier|
+            erb_modifier.add(modifier)
+          end
+          erb_modifier
+        end
+
+        private def create_config_file_modifier(workspace)
+          config_file_modifier = ConfigFileModifier.new(workspace)
+          Configs::create_config_file_modifiers.each do |modifier|
+            config_file_modifier.add(modifier)
+          end
+          config_file_modifier
+        end
+      end
+    end
+
+    ##
+    # Modifiers orchestrates all changes specified in the (json) config file.
+
+    class Modifiers
+
+      attr_reader :config_modifiers, :json_data
+
+      def initialize
+        @config_modifiers = Array.new
+        @json_path = nil
+        @json_data = nil
+      end
+
+      def with_json_configuration(json_file)
+        @json_path = json_file
+        self
+      end
+
+      def with_config_modifier(config_modifier)
+        if config_modifier.is_a? ConfigModifier
+          @config_modifiers << config_modifier
+        else
+          raise 'Parameter must be of type "ConfigModifier"'
+        end
+        self
+      end
+
+      def modify_configs
+        load_config
+        check_all
+        map_all
+        verify_all
+        modify_all
+        finish_all
+      end
+
+      private def load_config
+        if File.file?(@json_path)
+          File.open(@json_path, 'r+') do |f|
+            @json_data = JSON.parse(f.read)
+          end
+        else
+          raise "Json configuration file does not exist at '#{@json_path}!'"
+        end
+      end
+
+      private def verify_all
+        @config_modifiers.each(&:verify)
+      end
+
+      def check_all
+        @config_modifiers.each(&:check)
+      end
+
+      private def map_all
+        @config_modifiers.each do |mapper|
+          mapper.map(@json_data)
+        end
+      end
+
+      private def modify_all
+        @config_modifiers.each(&:modify)
+      end
+
+      private def finish_all
+        @config_modifiers.each(&:finish)
+      end
+
+    end
+
+    ##
+    # ConfigModifier (and derived classes) calls the necessary steps to modify pi-gen build sources.
+    # It triggers check, mapping, and modification steps for one build configuration change.
+
+    class ConfigModifier
+
+      def initialize(workspace)
+        @workspace = workspace
+        @configs = Array.new
+      end
+
+      def add(config)
+        @configs << config
+      end
+
+      def check
+        $logger.debug("No pre check specified")
+      end
+
+      def verify
+        @configs.each {|config| config.verify unless config.nil?}
+      end
+
+      def map(json_data)
+        @configs.each {|config| config.map(json_data) unless config.nil?}
+      end
+
+      def modify
+        @configs.each {|config| config.modify(@workspace) unless config.nil?}
+      end
+
+      def finish
+        $logger.debug("Skipped finish")
+      end
+
+    end
+
+    ##
+    # ConfigFileModifier creates a pi-gen config file
+
+    class ConfigFileModifier < ConfigModifier
+
+      def initialize(workspace)
+        super(workspace)
+        @cfg_lines = Array.new
+        @config_file = 'config'
+      end
+
+      def modify
+        @configs.each {|config| @cfg_lines << config.config_line unless config.nil?}
+      end
+
+      def finish
+        FileUtils.mkdir_p "#{@workspace}"
+        File.open(File.join("#{@workspace}", "#{@config_file}"), 'w+') do |file|
+          @cfg_lines.each {|cfg_line| file.puts(cfg_line)}
+        end
+      end
+
+    end
+
+    ##
+    # ERBConfigModifier creates configurations from erb templates
+
+    class ERBConfigModifier < ConfigModifier
+
+      def initialize(workspace)
+        super(workspace)
+      end
+
+      def check
+        @configs.each do |config|
+          template_path = config.template_path
+          output_path = config.relative_output_path
+          template = read_template(template_path)
+
+          rendered = render(template, config.get_binding)
+          original = ""
+          File.open(output_path, 'r+') do |f|
+            original = f.read
+          end
+
+          unless rendered.equal? original
+            $logger.warn "Check of sources with default configuration failed:\nrendered = \n\n #{rendered} \n\n != original = \n\n #{original}"
+          end
+
+        end
+      end
+
+      def modify
+        create_conf
+      end
+
+      private def create_conf
+
+        @configs.each do |config|
+          template_path = config.template_path
+          output_path = File.join(@workspace, config.relative_output_path)
+
+          template = read_template(template_path)
+
+          # ensure that the output_path is available
+          FileUtils.mkdir_p File.dirname output_path
+
+          File.open(output_path, 'w+') do |f|
+            f.write(render(template, config.get_binding))
+          end
+
+          $logger.info "Created config at #{output_path} from #{template_path}"
+        end
+      end
+
+      private def read_template(template_path)
+        template = ""
+        File.open(template_path.to_s, 'r+') do |f|
+          template = f.read
+        end
+        template
+      end
+
+      private def render(template, binding)
+        ERB.new(template).result(binding)
+      end
+
+    end
+
+  end
+end

data/pi_build_modifier/lib/pi_build_modifier/{locale → modify}/templates/00-debconf.erb

@@ -3,4 +3,4 @@
 locales	locales/locales_to_be_generated	multiselect	<%= @gen_locales.join(', ') %>
 # Default locale for the system environment:
 # Choices: None, C.UTF-8, en_GB.UTF-8
-locales	locales/default_environmentg_locale	    select	<%= @sys_locale %>
+locales	locales/default_environment_locale	select	<%= @sys_locale %>

data/pi_build_modifier/lib/pi_build_modifier/{boot-files → modify}/templates/07-resize-init.diff.erb

@@ -2,4 +2,4 @@
 +++ b/rootfs/boot/cmdline.txt
 @@ -1 +1 @@
 -dwc_otg.lpm_enable=0 console=serial0,115200 console=tty1 root=ROOTDEV rootfstype=ext4 elevator=deadline fsck.repair=yes rootwait
-+dwc_otg.lpm_enable=0 console=serial0,115200 console=tty1 root=ROOTDEV rootfstype=ext4 <%= @c_groups.join(' ') %> elevator=deadline fsck.repair=yes rootwait quiet init=/usr/lib/raspi-config/init_resize.sh
++dwc_otg.lpm_enable=0 console=serial0,115200 console=tty1 root=ROOTDEV rootfstype=ext4 <%= @cgroups.join(' ') %> elevator=deadline fsck.repair=yes rootwait quiet init=/usr/lib/raspi-config/init_resize.sh

data/pi_build_modifier/lib/pi_build_modifier/{net-tweaks → modify}/templates/wpa_supplicant.conf.erb

@@ -2,4 +2,4 @@ country=<%= @wpa_country %>
 ctrl_interface=DIR=/var/run/wpa_supplicant GROUP=netdev
 update_config=1
 
-<% @networks.each do |network|  %>network=<%= network %><% end %>
+<% @networks.each do |network|  %>network=<%= network %><% end %>

data/pi_build_modifier/lib/pi_build_modifier/{config → utils}/logex.rb

@@ -21,5 +21,9 @@
 require 'logger'
 
 module PiBuildModifier
-  $logger = Logger.new(STDOUT)
+  begin
+    $logger = Logger.new('/etc/pi_build_modifier/modifier.log')
+  rescue
+    $logger = Logger.new(STDOUT)
+  end
 end

data/pi_build_modifier/lib/pi_build_modifier/version.rb

@@ -19,5 +19,5 @@
 # SOFTWARE.
 
 module PiBuildModifier
-  VERSION = '0.4.0.pre.alpha'
+  VERSION = '0.5.0.pre.alpha'
 end

data/pi_build_modifier/spec/pi_build_modifier/modify/modify_spec.rb

@@ -0,0 +1,306 @@
+# Copyright (c) 2017-2019 Beate Ottenwälder
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require_relative '../../spec_helper'
+require 'pi_build_modifier/modify/modifiers'
+require 'pi_build_modifier/modify/configs'
+
+
+RSpec.describe PiBuildModifier::Modifiers do
+
+  context 'ModifiersBuilder' do
+
+    it 'creates Modifiers' do
+      modifier = PiBuildModifier::Modifiers::ModifiersBuilder::build_defaults(".", "no.cfg")
+      expect(modifier.config_modifiers).not_to be_nil
+      expect(modifier.config_modifiers).not_to be_a PiBuildModifier::Modifiers::Modifiers
+    end
+
+    it 'creates and adds erb and config file modifiers to the Modifiers' do
+      modifier = PiBuildModifier::Modifiers::ModifiersBuilder::build_defaults("no.cfg", ".")
+      expect(modifier.config_modifiers).to include(be_a PiBuildModifier::Modifiers::ERBConfigModifier)
+      expect(modifier.config_modifiers).to include(be_a PiBuildModifier::Modifiers::ConfigFileModifier)
+    end
+
+  end
+
+  context 'Modifiers' do
+
+    let(:tmp_cfg_file) {'tmp.json'}
+
+    before(:each) do
+      File.open(tmp_cfg_file, 'w') {|file| file.write('{"cgroups": {"memory" : true}}')}
+    end
+
+    after(:each) do
+      FileUtils.rm_rf tmp_cfg_file
+    end
+
+    it 'holds configuration modifiers' do
+      modifiers = PiBuildModifier::Modifiers::Modifiers.new
+      modifiers.with_config_modifier(PiBuildModifier::Modifiers::ConfigModifier.new("."))
+      expect(modifiers.config_modifiers).to include(be_a PiBuildModifier::Modifiers::ConfigModifier)
+    end
+
+    it 'reads configurations from files' do
+      modifiers = PiBuildModifier::Modifiers::Modifiers.new
+      modifiers.with_json_configuration(tmp_cfg_file).modify_configs
+      expect(modifiers.json_data.has_key?('cgroups')).to be true
+    end
+
+    it 'triggers checks of the configuration modifiers' do
+      test_modifier = PiBuildModifier::Modifiers::ConfigModifier.new(".")
+      expect(test_modifier).to receive(:check)
+      modifiers = PiBuildModifier::Modifiers::Modifiers.new
+      modifiers.with_config_modifier(test_modifier).with_json_configuration(tmp_cfg_file).modify_configs
+    end
+
+    it 'triggers verifications of the configuration modifiers' do
+      test_modifier = PiBuildModifier::Modifiers::ConfigModifier.new(".")
+      expect(test_modifier).to receive(:verify)
+      modifiers = PiBuildModifier::Modifiers::Modifiers.new
+      modifiers.with_config_modifier(test_modifier).with_json_configuration(tmp_cfg_file).modify_configs
+    end
+
+    it 'triggers mappings of the configuration modifiers' do
+      test_modifier = PiBuildModifier::Modifiers::ConfigModifier.new(".")
+      expect(test_modifier).to receive(:map)
+      modifiers = PiBuildModifier::Modifiers::Modifiers.new
+      modifiers.with_config_modifier(test_modifier).with_json_configuration(tmp_cfg_file).modify_configs
+    end
+
+    it 'triggers modifications of the configuration modifiers' do
+      test_modifier = PiBuildModifier::Modifiers::ConfigModifier.new(".")
+      expect(test_modifier).to receive(:modify)
+      modifiers = PiBuildModifier::Modifiers::Modifiers.new
+      modifiers.with_config_modifier(test_modifier).with_json_configuration(tmp_cfg_file).modify_configs
+    end
+
+    it 'triggers finishing steps of the configuration modifiers' do
+      test_modifier = PiBuildModifier::Modifiers::ConfigModifier.new(".")
+      expect(test_modifier).to receive(:finish)
+      modifiers = PiBuildModifier::Modifiers::Modifiers.new
+      modifiers.with_json_configuration(tmp_cfg_file).with_config_modifier(test_modifier).modify_configs
+    end
+
+  end
+
+  context 'SystemConfig' do
+    it 'handle the name config' do
+      configuration = PiBuildModifier::Configs::SystemConfig.new
+      configuration.map(JSON.parse '{"system": {"name" : "test"}}')
+      expect(configuration.name).to eq('test')
+    end
+    it 'handle the name config' do
+      configuration = PiBuildModifier::Configs::SystemConfig.new
+      configuration.map(JSON.parse '{"system": {"username" : "user", "password" : "pwd"}}')
+      expect(configuration.username).to eq('user')
+      expect(configuration.password).to eq('pwd')
+    end
+    it 'returns the config change w/o  username and password' do
+      configuration = PiBuildModifier::Configs::SystemConfig.new
+      configuration.map(JSON.parse '{"system": {"name" : "test"}}')
+      expect(configuration.config_line).to eq("IMG_NAME='test'")
+    end
+    it 'returns the config change w/  username and password' do
+      configuration = PiBuildModifier::Configs::SystemConfig.new
+      configuration.map(JSON.parse '{"system": {"username" : "user", "password" : "pwd"}}')
+      expect(configuration.config_line).to eq("IMG_NAME='custompi'\nFIRST_USER_NAME=user\nFIRST_USER_PASS=pwd")
+    end
+    it 'ensures that a name for the image is provided' do
+      configuration = PiBuildModifier::Configs::SystemConfig.new
+      configuration.map(JSON.parse '{"system": {"name" : ""}}')
+      expect {configuration.verify}.to raise_error("No valid image name string provided in the config file! E.g., it might be an empty string or not a string at all. Configured image name: ''.")
+    end
+    it 'ensures that the name for the image ca' do
+      configuration = PiBuildModifier::Configs::SystemConfig.new
+      configuration.map(JSON.parse '{"system": {"name" : 1234}}')
+      expect {configuration.verify}.to raise_error("No valid image name string provided in the config file! E.g., it might be an empty string or not a string at all. Configured image name: '1234'.")
+    end
+  end
+
+  context 'SshConfig' do
+    it 'handle the ssh config' do
+      configuration = PiBuildModifier::Configs::SshConfig.new
+      configuration.map(JSON.parse '{"ssh": {"enable" : true}}')
+      expect(configuration.enable).to be_truthy
+    end
+    it 'returns the config change' do
+      configuration = PiBuildModifier::Configs::SshConfig.new
+      configuration.map(JSON.parse '{"ssh": {"enable" : true}}')
+      expect(configuration.config_line).to eq("ENABLE_SSH=1")
+    end
+  end
+
+  context 'Boot' do
+
+    let(:empty_json_config) {JSON.parse '{}'}
+    let(:json_config) {JSON.parse '{ "cgroups": { "memory" : true }}'}
+    let(:disabled_json_config) {JSON.parse '{ "cgroups": { "memory" : false }}'}
+    let(:workspace) {File.join(File.dirname(__FILE__), '/workspace_boot')}
+
+    before(:each) do
+      FileUtils.mkdir_p workspace
+    end
+
+    after(:each) do
+      FileUtils.rm_rf workspace
+    end
+
+    it 'should create a cmdline.txt file with all cgroups enabled when they are specified' do
+      modifier = PiBuildModifier::Modifiers::ERBConfigModifier.new(workspace)
+      config = PiBuildModifier::Configs::Boot.new
+      modifier.add(config)
+      modifier.map(json_config)
+      modifier.modify
+      expect(Pathname.new(File.join(workspace, config.relative_output_path))).to be_file
+      expect(File.read(File.join(workspace, config.relative_output_path))).to match(/ cgroup_memory=1 /)
+    end
+
+    it 'should create a cmdline.txt file without cgroups enabled when they are disabled' do
+      modifier = PiBuildModifier::Modifiers::ERBConfigModifier.new(workspace)
+      config = PiBuildModifier::Configs::Boot.new
+      modifier.add(config)
+      modifier.map(disabled_json_config)
+      modifier.modify
+      expect(Pathname.new(File.join(workspace, config.relative_output_path))).to be_file
+      expect(File.read(File.join(workspace, config.relative_output_path))).to_not match(/ cgroup_memory=1 /)
+    end
+
+    it 'should create a cmdline.txt file without cgroups enabled when they are not specified' do
+      modifier = PiBuildModifier::Modifiers::ERBConfigModifier.new(workspace)
+      config = PiBuildModifier::Configs::Boot.new
+      modifier.add(config)
+      modifier.map(empty_json_config)
+      modifier.modify
+      expect(Pathname.new(File.join(workspace, config.relative_output_path))).to be_file
+      expect(File.read(File.join(workspace, config.relative_output_path))).to_not match(/ cgroup_memory=1 /)
+    end
+  end
+
+  context 'Type' do
+
+    let(:json_config) {JSON.parse '{ "system": { "type" : "lite" }}'}
+    let(:invalid_json_config) {JSON.parse '{ "system": { "type" : "image" }}'}
+    let(:workspace) {File.join(File.dirname(__FILE__), '/workspace_boot')}
+
+    before(:each) do
+      FileUtils.mkdir_p workspace
+    end
+
+    after(:each) do
+      FileUtils.rm_rf workspace
+    end
+
+    it 'ensures that a valid type is specified' do
+      modifier = PiBuildModifier::Modifiers::ConfigModifier.new(workspace)
+      config = PiBuildModifier::Configs::TypeConfig.new
+      modifier.add(config)
+      modifier.map(invalid_json_config)
+      modifier.modify
+
+      expect {modifier.verify}.to raise_error
+    end
+
+    it 'should create skip files' do
+      modifier = PiBuildModifier::Modifiers::ConfigModifier.new(workspace)
+      config = PiBuildModifier::Configs::TypeConfig.new
+      modifier.add(config)
+      modifier.map(json_config)
+      modifier.modify
+
+      expect(Pathname.new(File.join(workspace, '/stage3/SKIP'))).to be_file
+      expect(Pathname.new(File.join(workspace, '/stage4/SKIP'))).to be_file
+      expect(Pathname.new(File.join(workspace, '/stage5/SKIP'))).to be_file
+    end
+
+  end
+
+  context 'WiFi' do
+    let(:empty_json_config) {JSON.parse '{}'}
+    let(:json_config) {JSON.parse '{"wifi" : { "wpa_country" : "DE_test", "networks": [{ "ssid": "test_wpsk", "wpsk": "test_wpsk" }]}}'}
+    let(:workspace) {File.dirname(__FILE__) + '/workspace'}
+
+    before do
+      FileUtils.mkdir_p workspace
+    end
+
+    after do
+      FileUtils.rm_rf workspace
+    end
+
+    it 'should create a default wpa_supplicant.config when no change is specified in the json configuration file' do
+      modifier = PiBuildModifier::Modifiers::ERBConfigModifier.new(workspace)
+      config = PiBuildModifier::Configs::Wifi.new
+      modifier.add(config)
+      modifier.map(json_config)
+      modifier.modify
+      expect(Pathname.new(File.join(workspace, config.relative_output_path))).to be_file
+    end
+
+    it 'should create a wpa_supplicant.config with all changes that are specified in the json configuration file' do
+      modifier = PiBuildModifier::Modifiers::ERBConfigModifier.new(workspace)
+      config = PiBuildModifier::Configs::Wifi.new
+      modifier.add(config)
+      modifier.map(json_config)
+      modifier.modify
+
+      wpa_supplicant_file = File.read(File.join(workspace, config.relative_output_path))
+      expect(wpa_supplicant_file).to match(/network/)
+    end
+  end
+
+  context 'Locales' do
+    let(:empty_json_config) {JSON.parse '{}'}
+    let(:json_config) {JSON.parse '{  "locale": { "gen" : ["aaa", "bbb"], "sys" : "ccc" }}'}
+    let(:workspace) {File.join(File.dirname(__FILE__), '/workspace')}
+
+    before(:each) do
+      FileUtils.mkdir_p workspace
+    end
+
+    after(:each) do
+      FileUtils.rm_rf workspace
+    end
+
+    it 'should create a debconf file' do
+      modifier = PiBuildModifier::Modifiers::ERBConfigModifier.new(workspace)
+      config = PiBuildModifier::Configs::Locale.new
+      modifier.add(config)
+      modifier.map(json_config)
+      modifier.modify
+
+      expect(Pathname.new(File.join(workspace, config.relative_output_path))).to be_file
+      expect(File.read(File.join(workspace, config.relative_output_path))).to match(/aaa, bbb/)
+      expect(File.read(File.join(workspace, config.relative_output_path))).to match(/ccc/)
+    end
+
+    it 'should create a default debconf file' do
+      modifier = PiBuildModifier::Modifiers::ERBConfigModifier.new(workspace)
+      config = PiBuildModifier::Configs::Locale.new
+      modifier.add(config)
+      modifier.map(empty_json_config)
+      modifier.modify
+
+      expect(Pathname.new(File.join(workspace, config.relative_output_path))).to be_file
+    end
+  end
+
+end