build-tool 0.5.7 → 0.6.0.rc1

data/.gitignore

@@ -9,3 +9,4 @@ pkg/*
 lib/build-tool/cfg/parser.output
 lib/build-tool/cfg/parser.rb
 lib/build-tool/cfg/lexer.rb
+.bundle/

data/.rvmrc

@@ -0,0 +1 @@
+rvm ruby-1.9.3-p0@0.6-dev

data/.yardopts

@@ -0,0 +1 @@
+--charset=utf-8

data/Gemfile

@@ -0,0 +1,5 @@
+# A sample Gemfile
+source "http://rubygems.org"
+
+gemspec
+

data/Gemfile.lock

@@ -0,0 +1,56 @@
+PATH
+  remote: .
+  specs:
+    build-tool (0.6)
+      activerecord (>= 3.2.1)
+      ansi (>= 1.4.2)
+      grit (>= 2.4.1)
+      logging (>= 1.6.0)
+      sqlite3 (>= 1.3.5)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activemodel (3.2.1)
+      activesupport (= 3.2.1)
+      builder (~> 3.0.0)
+    activerecord (3.2.1)
+      activemodel (= 3.2.1)
+      activesupport (= 3.2.1)
+      arel (~> 3.0.0)
+      tzinfo (~> 0.3.29)
+    activesupport (3.2.1)
+      i18n (~> 0.6)
+      multi_json (~> 1.0)
+    ansi (1.4.2)
+    arel (3.0.0)
+    builder (3.0.0)
+    diff-lcs (1.1.3)
+    grit (2.4.1)
+      diff-lcs (~> 1.1)
+      mime-types (~> 1.15)
+    i18n (0.6.0)
+    little-plugger (1.1.3)
+    logging (1.7.0)
+      little-plugger (>= 1.1.3)
+    mime-types (1.17.2)
+    multi_json (1.1.0)
+    racc (1.4.7)
+    rake (0.9.2.2)
+    rexical (1.0.5)
+    sqlite3 (1.3.5)
+    turn (0.9.3)
+      ansi
+    tzinfo (0.3.31)
+    yard (0.7.5)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  build-tool!
+  racc (>= 1.4.7)
+  rake (>= 0.9.2)
+  rexical (>= 1.0.5)
+  turn (>= 0.9)
+  yard (>= 0.7.5)

data/History.txt

@@ -1,3 +1,67 @@
+== 0.6.0 ==
+
+- Features
+  - Add --ionice and --nice N options to set IO (idle) and CPU Priority.
+    > kde-build --ionice --nice 19 build ....
+  - When updating the source show the newly applied commits if -v is specified.
+    They will allways end up in the rebase logfile.
+  - Make it possible to specify all modules at once. 
+    kde-build fetch :all
+  - New command 'recipe edit'
+    - Opens the recipe override file in a editor.
+    - --global will open the recipe itself.
+    - --browser will open a file browser instead.
+  - New command 'module cleanup'
+    - Can remove the src and build dir of a module. Handle with care.
+  - The enabled/disabled state of features has been moved to the database.
+    - On the first start the current state will be migrated to the database.
+      AFTER that remove all 'feature enable|disable' lines from the override
+      file.
+    - command 'feature enable FEATURE' enables a feature.
+    - command 'feature disable FEATURE' disables a feature.
+  - The enabled/disabled state of modules has been moved to the database.
+    - On the first start the current state will be migrated to the database.
+      AFTER that remove all 'modules enable|disable' lines from the override
+      file.
+    - command 'module enable MODULE' enables a module.
+    - command 'module disable MODULE' disables a module and has options to
+      remove src and bld directory in one step.
+  - The 'recipe fetch incoming' command now works against a branch that has the
+    build-tool version encoded. This makes it possible to make a recipe that is
+    compatible to more than one version.
+  - The configuration values have been moved into the command line. A migration
+    will be done automatically on first run.
+    - 'configuration list' Will show the configuration values.
+    - 'configuration edit' will edit them.
+
+
+- Enhancements
+  - Most of the --help texts have been improved dramatically.
+  - Unit testing was reenabled again.
+  - Command: lsmod
+    - Sort output alphabetically by default
+    - With --broken only shows modules that failed.
+    - With --no-sort shows modules in order of declaration / build order.
+    - With --all shows disabled modules too.
+    - With --disabled shows only disabled modules.
+  - Command lsfeat
+    - Looks much more similar to the module list output.
+    - Shows only active features by default.
+    - With --all shows all features.
+  - The long description statement now accepts multiline string with """....""".
+  - Command: gc
+    - Cleanup the logfiles more thoroughly.
+  - Disabled modules are accepted when given explicit on the command line if
+    they are associated with a active feature. This makes it possible to build
+    disabled modules.
+
+
+- Bugfixes
+  - Change current directory to $HOME on start. The current working directory
+    could be removed while the command runs.
+
+
+
 == 0.5.7 ==
 
 - Features

data/README.txt

@@ -46,13 +46,6 @@ Currently the following recipes are provided:
 
 http://michael-jansen.biz/build-tool
 
-== REQUIREMENTS:
-
-* logging      >= 1.4.3
-* sequel       >= 3.18.0
-* sqlite3-ruby >= 1.3.1
-* ansi         >= 1.2.2
-
 == INSTALL:
 
   gem install build-tool

data/Rakefile

@@ -1,61 +1,13 @@
 # -*- ruby -*-
-require 'rubygems'
+require 'bundler/gem_tasks'
 
-# http://rubyforge.org/tracker/index.php?func=detail&aid=28920&group_id=1513&atid=5921
-begin
-    require 'psych'
-rescue ::LoadError
-end
-
-require 'hoe'
-require 'fileutils'
-require './lib/build-tool'
-
-Hoe.plugin :git
-Hoe.plugin :doofus
-Hoe.plugin :debugging
-Hoe.plugin :racc
-
-
-# Generate all the Rake tasks
-# Run 'rake -T' to see list of generated tasks (from gem root directory)
-Hoe.spec( 'build-tool' ) do
-
-  self.developer 'Michael Jansen', 'info@michael-jansen.biz'
-
-  self.version = BuildTool::VERSION
-
-  self.post_install_message = <<-EOS
-    To start with build-tool try the following commands:
-    > build-tool recipe add git://gitorious.org/build-tool/kde-trunk-recipe.git kde
-    > build-tool recipe list
-    > build-tool recipe install kde
-
-    For documentation see http://michael-jansen.biz/build-tool
-
-  EOS
-  self.rubyforge_name = self.name
-
-  self.extra_deps <<
-    ['logging', ">= 1.4.3"] <<
-    ['sequel', ">= 3.18.0"] <<
-    ['sqlite3', ">= 1.3.3" ] <<
-    ['ansi', '>= 1.2.2' ] <<
-    ['grit', ">= 2.4.1" ]
-
-  self.extra_dev_deps <<
-    ['hoe-debugging', ">= 1.0.1"] <<
-    ['hoe-doofus', ">= 1.0.0"] <<
-    ['hoe-git', ">= 1.3.0"] <<
-    ['racc', ">= 1.4.6"] <<
-    ['rexical', ">= 1.0.5"]
-
-  self.rex_flags = ""
+Dir['tasks/**/*.rake'].each { |t| load t }
 
+# A task to build all generated files.
+namespace :bt do
+    desc "Generate all needed files"
+    task :genall => [ :rexical, :racc ]
 end
 
-Dir['tasks/**/*.rake'].each { |t| load t }
-
-# Want other tests/tasks run by default
-# remove_task :default
-# task :default => [:spec, :features]
+# Make sure all files are generated when building the pkg
+task :build => 'bt:genall'

data/bin/build-tool

@@ -1,15 +1,18 @@
 #!/usr/bin/env ruby
+# -*- coding: UTF-8 -*-
 #
 #  Created by Michael Jansen on 2009-8-31.
 #  Copyright (c) 2009. All rights reserved.
 
+# BEGIN { $VERBOSE = true }
+
 require 'pathname'
 require 'rubygems'
 
 # libs relative to the link target
 root_directory = Pathname.new(__FILE__).realpath.dirname.dirname
 
-require root_directory.join( 'lib/build-tool')
+require root_directory.join( 'lib/build-tool').to_s
 require 'build-tool/application'
 
 exit BuildTool.main( $0, ARGV, root_directory );

data/build-tool.gemspec

@@ -0,0 +1,62 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path('../lib', __FILE__)
+require 'build-tool/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'build-tool'
+  s.version     = BuildTool::VERSION
+  s.authors     = ['Michael Jansen']
+  s.email       = ['info@michael-jansen.biz']
+  s.homepage    = 'http://michael-jansen.biz'
+  s.summary     = %q{A tool helping to download, configure and compile from sources.}
+  s.description = %q{
+    The tool helps 
+
+    ...
+  }
+
+  s.license = 'GPL-V2+'
+
+  # It only works with ruby 1.9
+  s.required_ruby_version = '>= 1.9.2'
+
+  # Additional requirements (verbose)
+  # s.requirements << ''
+
+  # Documentation
+  s.rubyforge_project = 'build-tool'
+  s.extra_rdoc_files = [ 'README.txt' ]
+
+  # Dependencies
+  s.add_dependency( 'logging', '>= 1.6.0' ) # Logging Support
+  s.add_dependency( 'activerecord', '>= 3.2.1' ) # Object Relation Model
+  s.add_dependency( 'sqlite3', '>= 1.3.5' ) # SQLite Database Support
+  s.add_dependency( 'ansi', '>= 1.4.2' )    # ANSI Codes at your Fingertips
+  s.add_dependency( 'grit', '>= 2.4.1' )    # Git Support
+
+  # Development Dependencies
+  s.add_development_dependency( 'racc', '>= 1.4.7' )    # A LALR(1) parser generator.
+  s.add_development_dependency( 'rexical', '>= 1.0.5')  # A lexical scanner generator
+  s.add_development_dependency( 'rake', '>= 0.9.2' )
+  s.add_development_dependency( 'yard', '>= 0.7.5' )
+  s.add_development_dependency( 'turn', '>= 0.9' )    # Pretty test output
+
+  # Post installation message
+  s.post_install_message = <<-EOF
+Thank you for using build-tool. For documentation see http://michael-jansen.biz/build-tool .
+Report bugs to kde (at) michael-jansen.biz. 
+
+To start with build-tool try the following commands:
+> build-tool recipe add git://gitorious.org/build-tool/kde-trunk-recipe.git kde
+> build-tool recipe list
+> build-tool recipe install kde
+
+  EOF
+
+  s.files         = `git ls-files`.split("\n")
+  s.files        += Dir.glob( 'lib/**/*.y'   ).map { |p| p.sub( /.y$/,   '.rb' ) }
+  s.files        += Dir.glob( 'lib/**/*.rex' ).map { |p| p.sub( /.rex$/, '.rb' ) }
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ['lib']
+end

data/db/migrations/20110703074000_add_command_logs.rb

@@ -0,0 +1,17 @@
+class AddCommandLogs < ActiveRecord::Migration
+
+    def up
+        create_table :command_logs do |t|
+            t.string   :command, :null => false
+            t.string :logdir
+            t.datetime :started_at
+            t.datetime :finished_at
+            t.integer  :state, :default => 0, :null => false
+        end
+    end
+
+    def down
+        drop_table :command_logs
+    end
+
+end

data/db/migrations/20110703075000_add_module_logs.rb

@@ -0,0 +1,20 @@
+class AddModuleLogs < ActiveRecord::Migration
+
+    def up
+        create_table :module_logs do |t|
+            t.integer :command_log_id, :null => false
+            t.string :module, :null => false
+            t.string :event, :size => 20, :null => false
+            t.string :logfile
+            t.datetime :started_at
+            t.datetime :finished_at
+            t.integer  :state, :default => 0, :null => false
+        end
+    end
+
+    def down
+        drop_table :module_events
+    end
+
+end
+

data/db/migrations/20110815170000_add_features.rb

@@ -0,0 +1,17 @@
+class AddFeatures < ActiveRecord::Migration
+
+    def up
+        create_table :features do |t|
+            t.string   :name, :null => false
+            t.integer  :parent_id
+            t.boolean  :active,         :default => nil
+        end
+        add_index :features, [ :parent_id, :name ], :unique => true
+    end
+
+    def down
+        remove_index :features, [ :parent_id, :name ]
+        drop_table :features
+    end
+
+end

data/db/migrations/20120103204700_add_modules.rb

@@ -0,0 +1,17 @@
+class AddModules < ActiveRecord::Migration
+
+    def up
+        create_table :modules do |t|
+            t.string   :name, :null => false
+            t.boolean  :active, :default => nil
+        end
+        add_index :modules, [ :name ], :unique => true
+    end
+
+    def down
+        remove_index :modules, [ :name ]
+        drop_table :modules
+    end
+
+end
+

data/db/migrations/20120106181200_add_settings.rb

@@ -0,0 +1,18 @@
+class AddSettings < ActiveRecord::Migration
+
+    def up
+        create_table :settings do |t|
+            t.string :name, :null => false
+            t.string :value
+        end
+
+        add_index :settings, :name
+    end
+
+    def down
+        remove_index :settings, :name
+        drop_table :settings
+    end
+
+end
+

data/lib/build-tool.rb

@@ -1,7 +1,6 @@
+# -*- coding: UTF-8 -*-
+
 $:.unshift(File.dirname(__FILE__)) unless
   $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
-module BuildTool
-  VERSION = '0.5.7'
-end
-
+require 'build-tool/version'

data/lib/build-tool/application.rb

@@ -1,20 +1,20 @@
+# -*- encoding: UTF-8 -*-
 
 require 'logging'
-Logging.init :debug, :trace, :verbose, :info, :warn, :error
+Logging.init :debug2, :debug, :trace, :verbose, :info, :warn, :error
 include Logging.globally
 
 require 'optparse'
 require 'erb'
 require 'yaml'
-require 'sequel'
-require 'sequel/extensions/migration'
 
 require 'mj/logging'
 
 require "build-tool/errors"
-require 'build-tool/recipe'
 require 'build-tool/singleton'
-require 'build-tool/cfg/parser'
+
+require 'active_record'
+require 'sqlite3'
 
 $noop = false
 
@@ -34,9 +34,8 @@ class Application < Singleton
         attr_reader :name
 
         def recipe
-            return @recipe if @recipe
-            configuration
-            @recipe
+            return @configuration.recipe if @configuration
+            configuration.recipe
         end
 
         def initialize( name, application_root )
@@ -44,22 +43,107 @@ def initialize( name, application_root )
             @application_root = Pathname.new( application_root ).expand_path
             @database = nil
             @name = File.basename( name ).sub( "-build", "" )
-            @recipe = nil
             @local_settings_file_path = local_configuration_dir.join("#{@name}.yaml")
         end
 
+        # Execute necessary migration steps
+        def migrate()
+
+            logger.debug( 'Checking for obsolete modules in db.' )
+            for mod in @configuration.modules do
+                if not mod.found_in_recipe
+                    logger.info 'Module %s no longer supported by recipe. Removing it from db.' % [ mod.name ]
+                    mod.destroy()
+                end
+            end
+
+            logger.debug( 'Checking for obsolete settings in db.' )
+            @configuration.settings.each do |n, s|
+                if not s.seen and not n.start_with? 'BUILD_TOOL.'
+                    logger.info 'Setting %s no longer supported by recipe. Removing it from db.' % [ s.name ]
+                    s.destroy()
+                end
+            end
+
+        end
+
         # Load the configuration
         def configuration
             return @configuration if @configuration
-            # First load the settings file
+
+            require 'build-tool/configuration'
+            require 'build-tool/recipe'
+
+            # Read the configuration from the database
+            @configuration = Configuration.new()
+            @configuration.load()
+
+            #
+            ### MIGRATION FROM 0.5 START
+            #
+            # First load the old settings file if it exists. For new style the settings are
+            # correctly loaded from the database already.
             file = local_settings_file_path
-            logger.debug "Loading settings file #{file}"
-            raise ConfigurationError, "Configuration File #{file} does not exists!" if !file.exist?
-            @settings = YAML.load_file( file )
-            # Then load the recipe
-            logger.debug "Loading configuration for #{name}"
-            @recipe = BuildTool::Recipe.new( @settings['RECIPE'] || name )
-            @configuration = @recipe.load( name, @settings['SETTINGS'] )
+            if file.exist?
+                logger.info "Loading old style settings file #{file}."
+                YAML.load( File.open( file, 'r:UTF-8' ) ).each do |n, v|
+                    case n 
+                    when 'RECIPE'
+                        n = "BUILD_TOOL.RECIPE"
+                        logger.debug( "  %s='%s'" % [ n, v ] )
+                        s = ( @configuration.settings[ n ] or Setting.new( :name => n ) )
+                        s.value = v
+                        @configuration.add_setting( s )
+                    when 'SETTINGS'
+                        v.each do |n, v|
+                            logger.debug( "  %s='%s'" % [ n, v ] )
+                            s = ( @configuration.settings[ n ] or Setting.new( :name => n ) )
+                            s.value = v
+                            @configuration.add_setting( s )
+                        end
+                    else
+                        logger.warn( 'Unknown setting %s found in %s' % [ n, file ] )
+                    end
+                end
+                @configuration.save()
+                # Now rename the old file.
+                file.rename( file.to_s + '_0.5' )
+                logger.info( "Renamed old style settings file to #{file}_0.5. Please delete later!" )
+            end
+            #
+            ### MIGRATION FROM 0.5 END
+            #
+
+
+            # Find the recipe. Will throw an exception if recipe is invalid.
+            recipe = BuildTool::Recipe.new( @configuration.settings['BUILD_TOOL.RECIPE'].value )
+            logger.debug( 'Determined associated recipe as %s ( %s )' % [ recipe.name, recipe.global_path ] )
+            @configuration.recipe = recipe
+
+            # Initialize the settings from the settings.yaml file from the recipe
+            logger.debug( 'Loading configuration values from recipe:settings.yaml' )
+            YAML.load( File.open( recipe.global_config_file_path( 'settings.yaml' ), 'r:UTF-8' ) ).each do |v|
+                name = v['name']
+                logger.debug2( '  - Setting %s' % name )
+                s = ( @configuration.settings[ name ] or Setting.new( :name => name ) )
+                s.description = v['description']
+                s.default = v['default']
+                s.seen = true
+                @configuration.add_setting( s )
+            end
+
+            # Load the recipe
+            logger.debug( 'Loading recipe %s' % recipe.name )
+            @configuration = recipe.load( name, @configuration )
+
+            # Save possible changes to the configuration.
+            @configuration.save
+
+            # Migrate the recipe (if necessary)
+            migrate()
+
+            # Finished.
+            @configuration
         end
 
         # Return the local configuration dir. Makes sure the directory exists
@@ -76,18 +160,19 @@ def local_configuration_dir
         # Open the database
         #
         # @return [Object] The database handle
-        def open_database
+        def open_database( name )
             path = local_configuration_dir.join( "#{name}.db" )
             logger.debug "Opening the database #{path}."
-            @database = Sequel.connect( "sqlite:///#{path.to_s}" )
-            @database.logger= MJ::Logging::LoggerAdapter.new( 'db' )
-            # Try to upgrade the database if necessary
-            Sequel::Migrator.apply( @database, application_root.join( 'db/migrations' ) )
+            ActiveRecord::Base.establish_connection(
+                :adapter => 'sqlite3',
+                :database => path.to_s )
+            ActiveRecord::Migrator.migrate(
+                [ application_root.join( 'db/migrations' ) ] )
         end
 
         # Close the database
         def close_database
-            @database.disconnect
+            ActiveRecord::Base.remove_connection()
             path = local_configuration_dir.join( "#{name}.db" )
             logger.debug "Closing the database #{path}."
         end
@@ -102,7 +187,7 @@ def main( args )
             # don't need a configuration. Help for example.
 
             # Initialize the database.
-            open_database()
+            open_database( name )
 
             begin
 
@@ -110,14 +195,9 @@ def main( args )
                 require 'build-tool/commands'
                 @cli = Commands::Shell.new
 
-                # Load the commands. This has to be done AFTER the database is opened because
-                # sequel expects that there is a database before a Sequel::Model is created.
-                #
-                # And the commands require all that database related stuff.
-                @cli.load_commands( @application_root.join( "lib/build-tool/commands" ), BuildTool::Commands )
-
                 # Execute the given command (or start the cli)
                 @cli.execute( args )
+
             ensure
                 # Make sure the database is closed correctly
                 close_database
@@ -135,15 +215,24 @@ def has_recipe?
 
 def BuildTool.main(name, args, root_directory)
 
+    Dir.chdir( ENV["HOME"] )
+
     begin
         # Setup logging
-        Logging.logger.root.level = :debug
+        Logging.logger.root.level = :debug2
+
         # Special case for the configurationParser. It is very verbose if active.
         Logging.logger['BuildTool::Cfg::Parser'].level = :info
-        Logging.logger['MJ::Logging::LoggerAdapter'].level = :warn
-        Logging.logger['root'].appenders = Logging.appenders.stdout(
+
+        # Same goes for the activerecord logger, :info shows migrations
+        Logging.logger['ActiveRecord'].level = :warn
+        ActiveRecord::Base.logger = ::Logging::Logger['ActiveRecord']
+
+        # By default write only :info messages to stdout
+        Logging.logger['root'].add_appenders( Logging.appenders.stdout(
             :layout => MJ::Logging::BasicLayout.new(),
-            :level  => :info)
+            :level  => :info) )
+
         # Create the application
         app = BuildTool::Application.new( name, root_directory )
         # Execute the application
@@ -152,11 +241,12 @@ def BuildTool.main(name, args, root_directory)
         logger.info "User interrupt!"
         return 0
     rescue BuildTool::Error => e
-        logger.error   e.message
-        logger.verbose e.backtrace.join("\n")
+        logger.error( e.message )
+        logger.verbose( e.backtrace.join("\n") )
         return -1
     rescue Exception => e
-        logger.error e
+        logger.error( e )
+        logger.verbose( e.backtrace.join("\n") )
         return -1
     end