rscons 1.3.0 → 1.4.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    OWUzMDNlZjIxODVkZjhkNzg4MjQyZWZkMmQ2YWFiNjNmZmQyOWQ2OA==
+    NWMyYzkzNjU2MTQ1MmRiOTFkMjhlMzVlMjYzM2Q0MTJkOWRjMWM5Yw==
   data.tar.gz: !binary |-
-    MWRhYzY1YTNlMjIyMDkyYzE4ZjZjMTU4YjVlZmFiMDk2MjBjYWRiNg==
+    M2QyYmEwN2U2MGUyNGJkYjI4ODVjYWU5MzUyNWRiZTEzMWYxNDE3Mw==
 SHA512:
   metadata.gz: !binary |-
-    ODQwNTE4YjI5MmRiMjljNTMzMjRkYzU2OGZkMGUwNTgzMThlZmQzYWZhN2U0
-    NjUyNDgyZDUzZGQwZDMwYmJmZTYzNjA2MGQxMjcxYzIyNmQxNzcyM2RhODU5
-    ODMyMjMyODk5MTI5MWEwMTFlYWRlMjJiZTkwMDJhN2EyMGY1N2I=
+    MzkyZDE0ZjQ2M2NlNTdmMTZiNzNmNDIwMjUwMmEwZjI1N2Y5Mjg3YTU0YjYy
+    MWQwYjgwOGM2MGY3NjA0ZjlmZGM5ZThhN2UyZWM1OGIwMzk5MjQ2YzFhZjA1
+    N2FhMWI2ZjY2Njk1YjkwOWUwNGI5MjQ4MGU2N2U2NjM5NDlkYWM=
   data.tar.gz: !binary |-
-    ZjBkZDAzNjk3ZDkyMWFmMGEzZTg1ZTZiZTIzNmNjMmM4OGFmYzczOGQ0NTIw
-    NTMxNjUyNThjNzQ4MmMxYWY5YjNhYTY3NzZkZjQ0MjNmOTY5NzkxN2U5ZjY4
-    YzQ2NTIyNDMwZGU1OGNiMmIzZTM0ZWM3ODQwY2IzN2E1NGJlNTM=
+    YWRmNzc3OGU0NmE3ZTUzOWZkYmM5MjQ5OGU0ZTA2MWM2MmRlNzAyMzRiZTQx
+    YTc2NjRjYmZmYjU0MjA2ZmUyNmY5ZGI4OTZjN2Q5MzcyYzQ1ZWQ3Njg5MTc3
+    ZmVlY2NiZmViYmQyOWFmM2Q3MWEzMWY5ZTRkMjQ4ODYwNGM1ZmY=

data/README.md

@@ -12,7 +12,7 @@ Add this line to your application's Gemfile:
 
 And then execute:
 
-    $ bundle
+    $ bundle install
 
 Or install it yourself as:
 
@@ -176,11 +176,14 @@ subset of build targets or source files.
 The `build_op` parameter to the build hook block is a Hash describing the
 build operation with the following keys:
 * `:builder` - `Builder` instance in use
+* `:env` - `Environment` calling the build hook; note that this may be
+  different from the Environment that the build hook was added to in the case
+  that the original Environment was cloned with build hooks!
 * `:target` - `String` name of the target file
 * `:sources` - `Array` of the source files
-* `:vars` - `Rscons::VarSet` containing the construction variables to use
-The build hook can overwrite entries in `build_op[:vars]` to alter the
-construction variables in use for this specific build operation.
+* `:vars` - `Rscons::VarSet` containing the construction variables to use.
+  The build hook can overwrite entries in `build_op[:vars]` to alter the
+  construction variables in use for this specific build operation.
 
 ### Example: Creating a static library
 
@@ -190,19 +193,96 @@ Rscons::Environment.new do |env|
 end
 ```
 
+### Example: Creating a C++ parser source from a Yacc/Bison input file
+
+```ruby
+Rscons::Environment.new do |env|
+  env.CFile("#{env.build_root}/parser.tab.cc", "parser.yy")
+end
+```
+
 ## Details
 
-### Default Builders
+### Builders
 
-Rscons ships with a number of default builders:
+Rscons ships with a number of builders:
 
+* CFile, which builds a C or C++ source file from a lex or yacc input file
+* Disassemble, which disassembles an object file to a disassembly listing
 * Library, which collects object files into a static library archive file
 * Object, which compiles source files to produce an object file
+* Preprocess, which invokes the C/C++ preprocessor on a source file
 * Program, which links object files to produce an executable
 
 If you want to create an Environment that does not contain any builders,
 you can use the `exclude_builders` key to the Environment constructor.
 
+#### CFile
+
+```ruby
+env.CFile(target, source)
+# Example
+env.CFile("parser.c", "parser.y")
+```
+
+The CFile builder will generate a C or C++ source file from a lex (.l, .ll)
+or yacc (.y, .yy) input file.
+
+#### Disassemble
+
+```ruby
+env.Disassemble(target, source)
+# Example
+env.Disassemble("module.dis", "module.o")
+```
+
+The Disassemble builder generates a disassembly listing using objdump from
+and object file.
+
+#### Library
+
+```ruby
+env.Library(target, sources)
+# Example
+env.Library("lib.a", Dir["src/**/*.c"])
+```
+
+The Library builder creates a static library archive from the given source
+files.
+
+#### Object
+
+```ruby
+env.Object(target, sources)
+# Example
+env.Object("module.o", "module.c")
+```
+
+The Object builder compiles the given sources to an object file.
+
+#### Preprocess
+
+```ruby
+env.Preprocess(target, source)
+# Example
+env.Preprocess("module-preprocessed.cc", "module.cc")
+```
+
+The Preprocess builder invokes either ${CC} or ${CXX} (depending on if the
+source contains an extension in ${CXXSUFFIX} or not) and writes the
+preprocessed output to the target file.
+
+#### Program
+
+```ruby
+env.Program(target, sources)
+# Example
+env.Program("myprog", Dir["src/**/*.cc"])
+```
+
+The Program builder compiles and links the given sources to an executable file.
+Object files or source files can be given as `sources`.
+
 ### Managing Environments
 
 An Rscons::Environment consists of:
@@ -214,10 +294,17 @@ An Rscons::Environment consists of:
 * a collection of targets to build
 * a collection of build hooks
 
-When cloning an environment, the construction variables and builders are
-cloned, but the new environment does not inherit any of the targets, build
+When cloning an environment, by default the construction variables and builders
+are cloned, but the new environment does not inherit any of the targets, build
 hooks, build directories, or the build root from the source environment.
 
+The set of environment attributes that are cloned is controllable via the
+`:clone` option to the `#clone` method.
+For example, `env.clone(clone: :all)` will include construction variables,
+builders, build hooks, build directories, and the build root.
+
+The set of pending targets is never cloned.
+
 Cloned environments contain "deep copies" of construction variables.
 For example, in:
 
@@ -233,8 +320,7 @@ cloned_env["CPPPATH"] << "three"
 ### Construction Variable Naming
 
 * uppercase strings - the default construction variables that Rscons uses
-* lowercase symbols - Rscons options
-* lowercase strings - reserved as user-defined construction variables
+* symbols, lowercase strings - reserved as user-defined construction variables
 
 ### API documentation
 
@@ -243,6 +329,22 @@ http://rubydoc.info/github/holtrop/rscons/frames.
 
 ## Release Notes
 
+### v1.4.0
+
+- add CFile builder
+- add Disassemble builder
+- add Preprocess builder
+- pass the Environment object to build hooks in the :env key of the build_op parameter
+- expand target/source paths beginning with "^/" to be relative to the Environment's build root
+- many performance improvements, including:
+  - use JSON instead of YAML for the cache to improve loading speed (Issue #7)
+  - store a hash of the build command instead of the full command contents in the cache
+  - implement copy-on-write semantics for construction variables when cloning Environments
+  - only load the cache once instead of on each Environment#process
+  - only write the cache when something has changed
+- fix Cache#mkdir_p to handle relative paths (Issue #5)
+- flush the cache to disk if a builder raises an exception (Issue #4)
+
 ### v1.3.0
 
 - change Environment#execute() options parameter to accept the following options keys:

data/lib/rscons.rb

@@ -5,15 +5,21 @@ require_relative "rscons/varset"
 require_relative "rscons/version"
 
 # default builders
+require_relative "rscons/builders/cfile"
+require_relative "rscons/builders/disassemble"
 require_relative "rscons/builders/library"
 require_relative "rscons/builders/object"
+require_relative "rscons/builders/preprocess"
 require_relative "rscons/builders/program"
 
 # Namespace module for rscons classes
 module Rscons
   DEFAULT_BUILDERS = [
+    :CFile,
+    :Disassemble,
     :Library,
     :Object,
+    :Preprocess,
     :Program,
   ]
 
@@ -21,7 +27,7 @@ module Rscons
 
   # Remove all generated files
   def self.clean
-    cache = Cache.new
+    cache = Cache.instance
     # remove all built files
     cache.targets.each do |target|
       FileUtils.rm_f(target)
@@ -33,7 +39,7 @@ module Rscons
         Dir.rmdir(directory) rescue nil
       end
     end
-    Cache.clear
+    cache.clear
   end
 
   # Return whether the given path is an absolute filesystem path or not

data/lib/rscons/builders/cfile.rb

@@ -0,0 +1,40 @@
+module Rscons
+  module Builders
+    # Build a C or C++ source file given a lex (.l, .ll) or yacc (.y, .yy)
+    # input file.
+    #
+    # Examples::
+    #   env.CFile("parser.tab.cc", "parser.yy")
+    #   env.CFile("lex.yy.cc", "parser.ll")
+    class CFile < Builder
+      def default_variables(env)
+        {
+          "YACC" => "bison",
+          "YACC_FLAGS" => ["-d"],
+          "YACC_CMD" => ["${YACC}", "${YACC_FLAGS}", "-o", "${_TARGET}", "${_SOURCES}"],
+          "LEX" => "flex",
+          "LEX_FLAGS" => [],
+          "LEX_CMD" => ["${LEX}", "${LEX_FLAGS}", "-o", "${_TARGET}", "${_SOURCES}"],
+        }
+      end
+
+      def run(target, sources, cache, env, vars)
+        vars = vars.merge({
+          "_TARGET" => target,
+          "_SOURCES" => sources,
+        })
+        cmd =
+          case
+          when sources.first.end_with?(".l", ".ll")
+            "LEX"
+          when sources.first.end_with?(".y", ".yy")
+            "YACC"
+          else
+            raise "Unknown source file #{sources.first.inspect} for CFile builder"
+          end
+        command = env.build_command(env["#{cmd}_CMD"], vars)
+        standard_build("#{cmd} #{target}", target, command, sources, env, cache)
+      end
+    end
+  end
+end

data/lib/rscons/builders/disassemble.rb

@@ -0,0 +1,25 @@
+module Rscons
+  module Builders
+    # The Disassemble builder produces a disassembly listing of a source file.
+    class Disassemble < Builder
+      def default_variables(env)
+        {
+          "OBJDUMP" => "objdump",
+          "DISASM_CMD" => ["${OBJDUMP}", "${DISASM_FLAGS}", "${_SOURCES}"],
+          "DISASM_FLAGS" => ["--disassemble", "--source"],
+        }
+      end
+
+      def run(target, sources, cache, env, vars)
+        vars = vars.merge("_SOURCES" => sources)
+        command = env.build_command(env["DISASM_CMD"], vars)
+        unless cache.up_to_date?(target, command, sources, env)
+          cache.mkdir_p(File.dirname(target))
+          return false unless env.execute("Disassemble #{target}", command, options: {out: target})
+          cache.register_build(target, command, sources, env)
+        end
+        target
+      end
+    end
+  end
+end

data/lib/rscons/builders/library.rb

@@ -1,7 +1,7 @@
 module Rscons
   module Builders
     # A default Rscons builder that produces a static library archive.
-    class Rscons::Builders::Library < Rscons::Builder
+    class Library < Builder
       def default_variables(env)
         {
           'AR' => 'ar',

data/lib/rscons/builders/preprocess.rb

@@ -0,0 +1,25 @@
+module Rscons
+  module Builders
+    # The Preprocess builder invokes the C preprocessor
+    class Preprocess < Builder
+      def default_variables(env)
+        {
+          "CPP_CMD" => ["${_PREPROCESS_CC}", "-E", "-o", "${_TARGET}", "-I${CPPPATH}", "${CPPFLAGS}", "${CFLAGS}", "${_SOURCES}"],
+        }
+      end
+
+      def run(target, sources, cache, env, vars)
+        pp_cc = if sources.find {|s| s.end_with?(*env["CXXSUFFIX"])}
+                  env["CXX"]
+                else
+                  env["CC"]
+                end
+        vars = vars.merge("_PREPROCESS_CC" => pp_cc,
+                          "_TARGET" => target,
+                          "_SOURCES" => sources)
+        command = env.build_command(env["CPP_CMD"], vars)
+        standard_build("Preprocess #{target}", target, command, sources, env, cache)
+      end
+    end
+  end
+end

data/lib/rscons/builders/program.rb

@@ -2,7 +2,7 @@ module Rscons
   module Builders
     # A default Rscons builder that knows how to link object files into an
     # executable program.
-    class Rscons::Builders::Program < Rscons::Builder
+    class Program < Builder
       def default_variables(env)
         {
           'LD' => nil,

data/lib/rscons/cache.rb

@@ -1,88 +1,87 @@
-require 'yaml'
-require 'fileutils'
-require 'digest/md5'
-require 'set'
-require 'rscons/version'
+require "digest/md5"
+require "fileutils"
+require "json"
+require "set"
+require "singleton"
+require "rscons/version"
 
 module Rscons
   # The Cache class keeps track of file checksums, build target commands and
-  # dependencies in a YAML file which persists from one invocation to the next.
+  # dependencies in a JSON file which persists from one invocation to the next.
   # Example cache:
   #   {
-  #     version: '1.2.3',
-  #     targets: {
-  #       'program' => {
-  #         'checksum' => 'A1B2C3D4',
-  #         'command' => ['gcc', '-o', 'program', 'program.o'],
-  #         'deps' => [
+  #     "version" => "1.2.3",
+  #     "targets" => {
+  #       "program" => {
+  #         "checksum" => "A1B2C3D4",
+  #         "command" => "13543518FE",
+  #         "deps" => [
   #           {
-  #             'fname' => 'program.o',
-  #             'checksum' => '87654321',
+  #             "fname" => "program.o",
+  #             "checksum" => "87654321",
   #           },
   #         ],
-  #         'user_deps' => [
+  #         "user_deps" => [
   #           {
-  #             'fname' => 'lscript.ld',
-  #             'checksum' => '77551133',
+  #             "fname" => "lscript.ld",
+  #             "checksum" => "77551133",
   #           },
   #         ],
   #       },
-  #       'program.o' => {
-  #         'checksum' => '87654321',
-  #         'command' => ['gcc', '-c', '-o', 'program.o', 'program.c'],
-  #         'deps' => [
+  #       "program.o" => {
+  #         "checksum" => "87654321",
+  #         "command" => "98765ABCD",
+  #         "deps" => [
   #           {
-  #             'fname' => 'program.c',
-  #             'checksum' => '456789ABC',
+  #             "fname" => "program.c",
+  #             "checksum" => "456789ABC",
   #           },
   #           {
-  #             'fname' => 'program.h',
-  #             'checksum' => '7979764643',
+  #             "fname" => "program.h",
+  #             "checksum" => "7979764643",
   #           },
   #         ],
-  #         'user_deps' => [],
+  #         "user_deps" => [],
   #       }
   #     },
-  #     directories: {
-  #       'build' => true,
-  #       'build/one' => true,
-  #       'build/two' => true,
+  #     "directories" => {
+  #       "build" => true,
+  #       "build/one" => true,
+  #       "build/two" => true,
   #     },
   #   }
   class Cache
-    #### Constants
+    include Singleton
 
     # Name of the file to store cache information in
-    CACHE_FILE = '.rsconscache'
+    CACHE_FILE = ".rsconscache"
 
-    #### Class Methods
+    # Create a Cache object and load in the previous contents from the cache
+    # file.
+    def initialize
+      initialize!
+    end
 
-    # Remove the cache file
-    def self.clear
+    # Remove the cache file.
+    def clear
       FileUtils.rm_f(CACHE_FILE)
+      initialize!
     end
 
-    #### Instance Methods
-
-    # Create a Cache object and load in the previous contents from the cache
-    # file.
-    def initialize
-      @cache = YAML.load(File.read(CACHE_FILE)) rescue {}
-      unless @cache.is_a?(Hash)
-        $stderr.puts "Warning: #{CACHE_FILE} was corrupt. Contents:\n#{@cache.inspect}"
-        @cache = {}
-      end
-      @cache[:targets] ||= {}
-      @cache[:directories] ||= {}
+    # Clear the cached file checksums.
+    def clear_checksum_cache!
       @lookup_checksums = {}
     end
 
     # Write the cache to disk to be loaded next time.
     def write
-      @cache[:version] = VERSION
-      File.open(CACHE_FILE, 'w') do |fh|
-        fh.puts(YAML.dump(@cache))
+      if @dirty || (@cache["version"] != VERSION)
+        @cache["version"] = VERSION
+        File.open(CACHE_FILE, "w") do |fh|
+          fh.puts(JSON.dump(@cache))
+        end
       end
+      @dirty = false
     end
 
     # Check if target(s) are up to date
@@ -90,11 +89,12 @@ module Rscons
     # @param command [String, Array] The command used to build the target.
     # @param deps [Array] List of the target's dependency files.
     # @param env [Environment] The Rscons::Environment.
-    # @param options [Hash] Optional options. Can contain the following keys:
-    #   :strict_deps::
-    #     Only consider a target up to date if its list of dependencies is
-    #     exactly equal (including order) to the cached list of dependencies
-    # @return true if the targets are all up to date, meaning that,
+    # @param options [Hash] Optional options.
+    # @option options [Boolean] :strict_deps
+    #   Only consider a target up to date if its list of dependencies is
+    #   exactly equal (including order) to the cached list of dependencies
+    # @return [Boolean]
+    #   True value if the targets are all up to date, meaning that,
     #   for each target:
     #   - the target exists on disk
     #   - the cache has information for the target
@@ -111,16 +111,16 @@ module Rscons
         return false unless File.exists?(target)
 
         # target must be registered in the cache
-        return false unless @cache[:targets].has_key?(target)
+        return false unless @cache["targets"].has_key?(target)
 
         # target must have the same checksum as when it was built last
-        return false unless @cache[:targets][target][:checksum] == lookup_checksum(target)
+        return false unless @cache["targets"][target]["checksum"] == lookup_checksum(target)
 
         # command used to build target must be identical
-        return false unless @cache[:targets][target][:command] == command
+        return false unless @cache["targets"][target]["command"] == Digest::MD5.hexdigest(command.inspect)
 
-        cached_deps = @cache[:targets][target][:deps] || []
-        cached_deps_fnames = cached_deps.map { |dc| dc[:fname] }
+        cached_deps = @cache["targets"][target]["deps"] || []
+        cached_deps_fnames = cached_deps.map { |dc| dc["fname"] }
         if options[:strict_deps]
           # depedencies passed in must exactly equal those in the cache
           return false unless deps == cached_deps_fnames
@@ -131,13 +131,13 @@ module Rscons
 
         # set of user dependencies must match
         user_deps = env.get_user_deps(target) || []
-        cached_user_deps = @cache[:targets][target][:user_deps] || []
-        cached_user_deps_fnames = cached_user_deps.map { |dc| dc[:fname] }
+        cached_user_deps = @cache["targets"][target]["user_deps"] || []
+        cached_user_deps_fnames = cached_user_deps.map { |dc| dc["fname"] }
         return false unless user_deps == cached_user_deps_fnames
 
         # all cached dependencies must have their checksums match
         (cached_deps + cached_user_deps).each do |dep_cache|
-          return false unless dep_cache[:checksum] == lookup_checksum(dep_cache[:fname])
+          return false unless dep_cache["checksum"] == lookup_checksum(dep_cache["fname"])
         end
       end
 
@@ -148,54 +148,70 @@ module Rscons
     # @param targets [String, Array] The name of the target(s) built.
     # @param command [String, Array] The command used to build the target.
     # @param deps [Array] List of dependencies for the target.
-    # @param env [Environment] The Rscons::Environment.
+    # @param env [Environment] The {Rscons::Environment}.
     def register_build(targets, command, deps, env)
       Array(targets).each do |target|
-        @cache[:targets][target.encode(__ENCODING__)] = {
-          command: command,
-          checksum: calculate_checksum(target),
-          deps: deps.map do |dep|
+        @cache["targets"][target] = {
+          "command" => Digest::MD5.hexdigest(command.inspect),
+          "checksum" => calculate_checksum(target),
+          "deps" => deps.map do |dep|
             {
-              fname: dep.encode(__ENCODING__),
-              checksum: lookup_checksum(dep),
+              "fname" => dep,
+              "checksum" => lookup_checksum(dep),
             }
           end,
-          user_deps: (env.get_user_deps(target) || []).map do |dep|
+          "user_deps" => (env.get_user_deps(target) || []).map do |dep|
             {
-              fname: dep.encode(__ENCODING__),
-              checksum: lookup_checksum(dep),
+              "fname" => dep,
+              "checksum" => lookup_checksum(dep),
             }
           end,
         }
+        @dirty = true
       end
     end
 
     # Return a list of targets that have been built
     def targets
-      @cache[:targets].keys
+      @cache["targets"].keys
     end
 
     # Make any needed directories and record the ones that are created for
     # removal upon a "clean" operation.
     def mkdir_p(path)
       parts = path.split(/[\\\/]/)
-      (0..parts.size-1).each do |i|
-        subpath = File.join(*parts[0, i + 1]).encode(__ENCODING__)
+      parts.each_index do |i|
+        next if parts[i] == ""
+        subpath = File.join(*parts[0, i + 1])
         unless File.exists?(subpath)
           FileUtils.mkdir(subpath)
-          @cache[:directories][subpath] = true
+          @cache["directories"][subpath] = true
+          @dirty = true
         end
       end
     end
 
     # Return a list of directories which were created as a part of the build
     def directories
-      @cache[:directories].keys
+      @cache["directories"].keys
     end
 
-    # Private Instance Methods
     private
 
+    # Create a Cache object and load in the previous contents from the cache
+    # file.
+    def initialize!
+      @cache = JSON.load(File.read(CACHE_FILE)) rescue {}
+      unless @cache.is_a?(Hash)
+        $stderr.puts "Warning: #{CACHE_FILE} was corrupt. Contents:\n#{@cache.inspect}"
+        @cache = {}
+      end
+      @cache["targets"] ||= {}
+      @cache["directories"] ||= {}
+      @lookup_checksums = {}
+      @dirty = false
+    end
+
     # Return a file's checksum, or the previously calculated checksum for
     # the same file
     # @param file [String] The file name.
@@ -206,7 +222,7 @@ module Rscons
     # Calculate and return a file's checksum
     # @param file [String] The file name.
     def calculate_checksum(file)
-      @lookup_checksums[file] = Digest::MD5.hexdigest(File.read(file, mode: 'rb')).encode(__ENCODING__) rescue ''
+      @lookup_checksums[file] = Digest::MD5.hexdigest(File.read(file, mode: "rb")) rescue ""
     end
   end
 end