arduino_ci 0.1.20 → 0.4.0

data/lib/arduino_ci.rb

@@ -2,6 +2,7 @@ require "arduino_ci/version"
 require "arduino_ci/arduino_installation"
 require "arduino_ci/cpp_library"
 require "arduino_ci/ci_config"
+require "arduino_ci/library_properties"
 
 # ArduinoCI contains classes for automated testing of Arduino code on the command line
 # @author Ian Katz <ianfixes@gmail.com>

data/lib/arduino_ci/arduino_cmd.rb

@@ -6,6 +6,9 @@ WORKAROUND_LIB = "USBHost".freeze
 
 module ArduinoCI
 
+  # To report errors that we can't resolve or possibly even explain
+  class ArduinoExecutionError < StandardError; end
+
   # Wrap the Arduino executable.  This requires, in some cases, a faked display.
   class ArduinoCmd
 
@@ -42,14 +45,14 @@ module ArduinoCI
     attr_reader   :last_msg
 
     # set the command line flags (undefined for now).
-    # These vary between gui/cli
-    flag :get_pref
-    flag :set_pref
-    flag :save_prefs
-    flag :use_board
-    flag :install_boards
-    flag :install_library
-    flag :verify
+    # These vary between gui/cli.  Inline comments added for greppability
+    flag :get_pref           # flag_get_pref
+    flag :set_pref           # flag_set_pref
+    flag :save_prefs         # flag_save_prefs
+    flag :use_board          # flag_use_board
+    flag :install_boards     # flag_install_boards
+    flag :install_library    # flag_install_library
+    flag :verify             # flag_verify
 
     def initialize
       @prefs_cache        = {}
@@ -82,7 +85,8 @@ module ArduinoCI
     # @return [String] Preferences as a set of lines
     def _prefs_raw
       resp = run_and_capture(flag_get_pref)
-      return nil unless resp[:success]
+      fail_msg = "Arduino binary failed to operate as expected; you will have to troubleshoot it manually"
+      raise ArduinoExecutionError, "#{fail_msg}. The command was #{@last_msg}" unless resp[:success]
 
       @prefs_fetched = true
       resp[:out]

data/lib/arduino_ci/arduino_downloader.rb

@@ -161,16 +161,17 @@ module ArduinoCI
         return false
       end
 
+      arduino_package = "Arduino #{@desired_ide_version} package"
       attempts = 0
 
       loop do
         if File.exist? package_file
-          @output.puts "Arduino package seems to have been downloaded already" if attempts.zero?
+          @output.puts "#{arduino_package} seems to have been downloaded already" if attempts.zero?
           break
         elsif attempts >= DOWNLOAD_ATTEMPTS
           break @output.puts "After #{DOWNLOAD_ATTEMPTS} attempts, failed to download #{package_url}"
         else
-          @output.print "Attempting to download Arduino package with #{downloader}"
+          @output.print "Attempting to download #{arduino_package} with #{downloader}"
           download
           @output.puts
         end
@@ -178,7 +179,7 @@ module ArduinoCI
       end
 
       if File.exist? extracted_file
-        @output.puts "Arduino package seems to have been extracted already"
+        @output.puts "#{arduino_package} seems to have been extracted already"
       elsif File.exist? package_file
         @output.print "Extracting archive with #{extracter}"
         extract
@@ -186,7 +187,7 @@ module ArduinoCI
       end
 
       if File.exist? self.class.force_install_location
-        @output.puts "Arduino package seems to have been installed already"
+        @output.puts "#{arduino_package} seems to have been installed already"
       elsif File.exist? extracted_file
         install
       else

data/lib/arduino_ci/arduino_installation.rb

@@ -110,11 +110,11 @@ module ArduinoCI
       # Forcibly install Arduino from the web
       # @return [bool] Whether the command succeeded
       def force_install(output = $stdout, version = DESIRED_ARDUINO_IDE_VERSION)
-        worker_class =  case Host.os
-                        when :osx then ArduinoDownloaderOSX
-                        when :windows then ArduinoDownloaderWindows
-                        when :linux then ArduinoDownloaderLinux
-                        end
+        worker_class = case Host.os
+        when :osx then ArduinoDownloaderOSX
+        when :windows then ArduinoDownloaderWindows
+        when :linux then ArduinoDownloaderLinux
+        end
         worker = worker_class.new(version, output)
         worker.execute
       end

data/lib/arduino_ci/ci_config.rb

@@ -28,6 +28,7 @@ UNITTEST_SCHEMA = {
   compilers: Array,
   platforms: Array,
   libraries: Array,
+  exclude_dirs: Array,
   testfiles: {
     select: Array,
     reject: Array,
@@ -256,6 +257,14 @@ module ArduinoCI
       @unittest_info[:compilers]
     end
 
+    # paths to exclude all files in for building and unitttests
+    # @return [Array<String>] The directories (relative to base dir) to exclude
+    def exclude_dirs
+      return [] if @unittest_info[:exclude_dirs].nil?
+
+      @unittest_info[:exclude_dirs]
+    end
+
     # platforms to build [the examples on]
     # @return [Array<String>] The platforms to build
     def platforms_to_build
@@ -293,9 +302,12 @@ module ArduinoCI
       return paths if @unittest_info[:testfiles].nil?
 
       ret = paths
+      # Check for array emptiness, otherwise nothing will be selected!
       unless @unittest_info[:testfiles][:select].nil? || @unittest_info[:testfiles][:select].empty?
         ret.select! { |p| unittest_info[:testfiles][:select].any? { |glob| p.basename.fnmatch(glob) } }
       end
+
+      # It's OK for the :reject array to be empty, that means nothing will be rejected by default
       unless @unittest_info[:testfiles][:reject].nil?
         ret.reject! { |p| unittest_info[:testfiles][:reject].any? { |glob| p.basename.fnmatch(glob) } }
       end

data/lib/arduino_ci/cpp_library.rb

@@ -37,11 +37,15 @@ module ArduinoCI
 
     # @param base_dir [Pathname] The path to the library being tested
     # @param arduino_lib_dir [Pathname] The path to the libraries directory
-    def initialize(base_dir, arduino_lib_dir)
+    # @param exclude_dirs [Array<Pathname>] Directories that should be excluded from compilation
+    def initialize(base_dir, arduino_lib_dir, exclude_dirs)
       raise ArgumentError, 'base_dir is not a Pathname' unless base_dir.is_a? Pathname
       raise ArgumentError, 'arduino_lib_dir is not a Pathname' unless arduino_lib_dir.is_a? Pathname
+      raise ArgumentError, 'exclude_dir is not an array of Pathnames' unless exclude_dirs.is_a?(Array)
+      raise ArgumentError, 'exclude_dir array contains non-Pathname elements' unless exclude_dirs.all? { |p| p.is_a? Pathname }
 
       @base_dir = base_dir
+      @exclude_dirs = exclude_dirs
       @arduino_lib_dir = arduino_lib_dir.expand_path
       @artifacts = []
       @last_err = ""
@@ -51,6 +55,32 @@ module ArduinoCI
       @vendor_bundle_cache = nil
     end
 
+    # The expected path to the library.properties file (i.e. even if it does not exist)
+    # @return [Pathname]
+    def library_properties_path
+      @base_dir + "library.properties"
+    end
+
+    # Whether library.properties definitions for this library exist
+    # @return [bool]
+    def library_properties?
+      lib_props = library_properties_path
+      lib_props.exist? && lib_props.file?
+    end
+
+    # Decide whether this is a 1.5-compatible library
+    #
+    # according to https://arduino.github.io/arduino-cli/latest/library-specification
+    #
+    # Should match logic from https://github.com/arduino/arduino-cli/blob/master/arduino/libraries/loader.go
+    # @return [bool]
+    def one_point_five?
+      return false unless library_properties?
+
+      src_dir = (@base_dir + "src")
+      src_dir.exist? && src_dir.directory?
+    end
+
     # Guess whether a file is part of the vendor bundle (indicating we should ignore it).
     #
     # A safe way to do this seems to be to check whether any of the installed gems
@@ -106,6 +136,8 @@ module ArduinoCI
     # @param path [Pathname] The path to check
     # @return [bool]
     def in_tests_dir?(path)
+      return false unless tests_dir.exist?
+
       tests_dir_aliases = [tests_dir, tests_dir.realpath]
       # we could do this but some rubies don't return an enumerator for ascend
       # path.ascend.any? { |part| tests_dir_aliases.include?(part) }
@@ -115,6 +147,19 @@ module ArduinoCI
       false
     end
 
+    # Guess whether a file is part of any @excludes_dir dir (indicating library compilation should ignore it).
+    #
+    # @param path [Pathname] The path to check
+    # @return [bool]
+    def in_exclude_dir?(path)
+      # we could do this but some rubies don't return an enumerator for ascend
+      # path.ascend.any? { |part| tests_dir_aliases.include?(part) }
+      path.ascend do |part|
+        return true if exclude_dir.any? { |p| p.realpath == part }
+      end
+      false
+    end
+
     # Check whether libasan (and by extension -fsanitizer=address) is supported
     #
     # This requires compilation of a sample program, and will be cached
@@ -133,43 +178,98 @@ module ArduinoCI
       @has_libasan_cache[gcc_binary]
     end
 
+    # Library properties
+    def library_properties
+      return nil unless library_properties?
+
+      LibraryProperties.new(library_properties_path)
+    end
+
+    # Get a list of all dependencies as defined in library.properties
+    # @return [Array<String>] The library names of the dependencies (not the paths)
+    def arduino_library_dependencies
+      return nil unless library_properties?
+
+      library_properties.depends
+    end
+
     # Get a list of all CPP source files in a directory and its subdirectories
     # @param some_dir [Pathname] The directory in which to begin the search
+    # @param extensions [Array<Sring>] The set of allowable file extensions
     # @return [Array<Pathname>] The paths of the found files
-    def cpp_files_in(some_dir)
+    def code_files_in(some_dir, extensions)
       raise ArgumentError, 'some_dir is not a Pathname' unless some_dir.is_a? Pathname
       return [] unless some_dir.exist? && some_dir.directory?
 
-      real = some_dir.realpath
-      files = Find.find(real).map { |p| Pathname.new(p) }.reject(&:directory?)
-      cpp = files.select { |path| CPP_EXTENSIONS.include?(path.extname.downcase) }
+      files = some_dir.realpath.children.reject(&:directory?)
+      cpp = files.select { |path| extensions.include?(path.extname.downcase) }
       not_hidden = cpp.reject { |path| path.basename.to_s.start_with?(".") }
       not_hidden.sort_by(&:to_s)
     end
 
+    # Get a list of all CPP source files in a directory and its subdirectories
+    # @param some_dir [Pathname] The directory in which to begin the search
+    # @param extensions [Array<Sring>] The set of allowable file extensions
+    # @return [Array<Pathname>] The paths of the found files
+    def code_files_in_recursive(some_dir, extensions)
+      raise ArgumentError, 'some_dir is not a Pathname' unless some_dir.is_a? Pathname
+      return [] unless some_dir.exist? && some_dir.directory?
+
+      real = some_dir.realpath
+      Find.find(real).map { |p| Pathname.new(p) }.select(&:directory?).map { |d| code_files_in(d, extensions) }.flatten
+    end
+
+    # Header files that are part of the project library under test
+    # @return [Array<Pathname>]
+    def header_files
+      ret = if one_point_five?
+        code_files_in_recursive(@base_dir + "src", HPP_EXTENSIONS)
+      else
+        [@base_dir, @base_dir + "utility"].map { |d| code_files_in(d, HPP_EXTENSIONS) }.flatten
+      end
+
+      # note to future troubleshooter: some of these tests may not be relevant, but at the moment at
+      # least some of them are tied to existing features
+      ret.reject { |p| vendor_bundle?(p) || in_tests_dir?(p) || in_exclude_dir?(p) }
+    end
+
     # CPP files that are part of the project library under test
     # @return [Array<Pathname>]
     def cpp_files
-      cpp_files_in(@base_dir).reject { |p| vendor_bundle?(p) || in_tests_dir?(p) }
+      ret = if one_point_five?
+        code_files_in_recursive(@base_dir + "src", CPP_EXTENSIONS)
+      else
+        [@base_dir, @base_dir + "utility"].map { |d| code_files_in(d, CPP_EXTENSIONS) }.flatten
+      end
+
+      # note to future troubleshooter: some of these tests may not be relevant, but at the moment at
+      # least some of them are tied to existing features
+      ret.reject { |p| vendor_bundle?(p) || in_tests_dir?(p) || in_exclude_dir?(p) }
     end
 
     # CPP files that are part of the arduino mock library we're providing
     # @return [Array<Pathname>]
     def cpp_files_arduino
-      cpp_files_in(ARDUINO_HEADER_DIR)
+      code_files_in(ARDUINO_HEADER_DIR, CPP_EXTENSIONS)
     end
 
     # CPP files that are part of the unit test library we're providing
     # @return [Array<Pathname>]
     def cpp_files_unittest
-      cpp_files_in(UNITTEST_HEADER_DIR)
+      code_files_in(UNITTEST_HEADER_DIR, CPP_EXTENSIONS)
     end
 
     # CPP files that are part of the 3rd-party libraries we're including
     # @param [Array<String>] aux_libraries
     # @return [Array<Pathname>]
     def cpp_files_libraries(aux_libraries)
-      arduino_library_src_dirs(aux_libraries).map { |d| cpp_files_in(d) }.flatten.uniq
+      arduino_library_src_dirs(aux_libraries).map { |d| code_files_in(d, CPP_EXTENSIONS) }.flatten.uniq
+    end
+
+    # Returns the Pathnames for all paths to exclude from testing and compilation
+    # @return [Array<Pathname>]
+    def exclude_dir
+      @exclude_dirs.map { |p| Pathname.new(@base_dir) + p }.select(&:exist?)
     end
 
     # The directory where we expect to find unit test defintions provided by the user
@@ -181,16 +281,15 @@ module ArduinoCI
     # The files provided by the user that contain unit tests
     # @return [Array<Pathname>]
     def test_files
-      cpp_files_in(tests_dir)
+      code_files_in(tests_dir, CPP_EXTENSIONS)
     end
 
     # Find all directories in the project library that include C++ header files
     # @return [Array<Pathname>]
     def header_dirs
-      real = @base_dir.realpath
-      all_files = Find.find(real).map { |f| Pathname.new(f) }.reject(&:directory?)
-      unbundled = all_files.reject { |path| vendor_bundle?(path) }
-      files = unbundled.select { |path| HPP_EXTENSIONS.include?(path.extname.downcase) }
+      unbundled = header_files.reject { |path| vendor_bundle?(path) }
+      unexcluded = unbundled.reject { |path| in_exclude_dir?(path) }
+      files = unexcluded.select { |path| HPP_EXTENSIONS.include?(path.extname.downcase) }
       files.map(&:dirname).uniq
     end
 
@@ -212,20 +311,24 @@ module ArduinoCI
       @last_err
     end
 
-    # Arduino library directories containing sources
+    # Arduino library directories containing sources -- only those of the dependencies
     # @return [Array<Pathname>]
     def arduino_library_src_dirs(aux_libraries)
       # Pull in all possible places that headers could live, according to the spec:
       # https://github.com/arduino/Arduino/wiki/Arduino-IDE-1.5:-Library-specification
-      # TODO: be smart and implement library spec (library.properties, etc)?
-      subdirs = ["", "src", "utility"]
-      all_aux_include_dirs_nested = aux_libraries.map do |libdir|
-        subdirs.map { |subdir| Pathname.new(@arduino_lib_dir) + libdir + subdir }
-      end
-      all_aux_include_dirs_nested.flatten.select(&:exist?).select(&:directory?)
+
+      aux_libraries.map do |d|
+        # library manager coerces spaces in package names to underscores
+        # see https://github.com/ianfixes/arduino_ci/issues/132#issuecomment-518857059
+        legal_dir = d.tr(" ", "_")
+        self.class.new(@arduino_lib_dir + legal_dir, @arduino_lib_dir, @exclude_dirs).header_dirs
+      end.flatten.uniq
     end
 
     # GCC command line arguments for including aux libraries
+    #
+    # This function recursively collects the library directores of the dependencies
+    #
     # @param aux_libraries [Array<Pathname>] The external Arduino libraries required by this project
     # @return [Array<String>] The GCC command-line flags necessary to include those libraries
     def include_args(aux_libraries)
@@ -288,6 +391,9 @@ module ArduinoCI
     end
 
     # build a file for running a test of the given unit test file
+    #
+    # The dependent libraries configuration is appended with data from library.properties internal to the library under test
+    #
     # @param test_file [Pathname] The path to the file containing the unit tests
     # @param aux_libraries [Array<Pathname>] The external Arduino libraries required by this project
     # @param ci_gcc_config [Hash] The GCC config object
@@ -306,8 +412,12 @@ module ArduinoCI
           "-fsanitize=address"
         ]
       end
-      arg_sets << test_args(aux_libraries, ci_gcc_config)
-      arg_sets << cpp_files_libraries(aux_libraries).map(&:to_s)
+
+      # combine library.properties defs (if existing) with config file.
+      # TODO: as much as I'd like to rely only on the properties file(s), I think that would prevent testing 1.0-spec libs
+      full_aux_libraries = arduino_library_dependencies.nil? ? aux_libraries : aux_libraries + arduino_library_dependencies
+      arg_sets << test_args(full_aux_libraries, ci_gcc_config)
+      arg_sets << cpp_files_libraries(full_aux_libraries).map(&:to_s)
       arg_sets << [test_file.to_s]
       args = arg_sets.flatten(1)
       return nil unless run_gcc(gcc_binary, *args)
@@ -316,14 +426,31 @@ module ArduinoCI
       executable
     end
 
+    # print any found stack dumps
+    # @param executable [Pathname] the path to the test file
+    def print_stack_dump(executable)
+      possible_dumpfiles = [
+        executable.sub_ext(executable.extname + ".stackdump")
+      ]
+      possible_dumpfiles.select(&:exist?).each do |dump|
+        puts "========== Stack dump from #{dump}:"
+        File.foreach(dump) { |line| print "    #{line}" }
+      end
+    end
+
     # run a test file
-    # @param [Pathname] the path to the test file
+    # @param executable [Pathname] the path to the test file
     # @return [bool] whether all tests were successful
     def run_test_file(executable)
       @last_cmd = executable
       @last_out = ""
       @last_err = ""
-      Host.run_and_output(executable.to_s.shellescape)
+      ret = Host.run_and_output(executable.to_s.shellescape)
+
+      # print any stack traces found during a failure
+      print_stack_dump(executable) unless ret
+
+      ret
     end
 
   end

data/lib/arduino_ci/library_properties.rb

@@ -0,0 +1,86 @@
+module ArduinoCI
+
+  # Information about an Arduino library package, as specified by the library.properties file
+  #
+  # See https://arduino.github.io/arduino-cli/library-specification/#libraryproperties-file-format
+  class LibraryProperties
+
+    # @return [Hash] The properties file parsed as a hash
+    attr_reader :fields
+
+    # @param path [Pathname] The path to the library.properties file
+    def initialize(path)
+      @fields = {}
+      File.foreach(path) do |line|
+        parts = line.split("=", 2)
+        @fields[parts[0]] = parts[1].chomp unless parts.empty?
+      end
+    end
+
+    # Enable a shortcut syntax for library property accessors, in the style of `attr_accessor` metaprogramming.
+    # This is used to create a named field pointing to a specific property in the file, optionally applying
+    # a specific formatting function.
+    #
+    #  The formatting function MUST be a static method on this class.  This is a limitation caused by the desire
+    #  to both (1) expose the formatters outside this class, and (2) use them for metaprogramming without the
+    #  having to name the entire function.  field_reader is a static method, so if not for the fact that
+    #  `self.class.methods.include? formatter` fails to work for class methods in this context (unlike
+    #  `self.methods.include?`, which properly finds instance methods), I would allow either one and just
+    #  conditionally `define_method` the proper definition
+    #
+    # @param name [String] What the accessor will be called
+    # @param field_num [Integer] The name of the key of the property
+    # @param formatter [Symbol] The symbol for the formatting function to apply to the field (optional)
+    # @return [void]
+    # @macro [attach] field_reader
+    #   @!attribute [r] $1
+    #   @return property $2 of the library.properties file, formatted with the function {$3}
+    def self.field_reader(name, formatter = nil)
+      key = name.to_s
+      if formatter.nil?
+        define_method(name) { @fields[key] }
+      else
+        define_method(name) { @fields.key?(key) ? self.class.send(formatter.to_sym, @fields[key]) : nil }
+      end
+    end
+
+    # Parse a value as a comma-separated array
+    # @param input [String]
+    # @return [Array<String>] The individual values
+    def self._csv(input)
+      input.split(",").map(&:strip)
+    end
+
+    # Parse a value as a boolean
+    # @param input [String]
+    # @return [Array<String>] The individual values
+    def self._bool(input)
+      input == "true"  # no indication given in the docs that anything but lowercase "true" indicates boolean true.
+    end
+
+    field_reader :name
+    field_reader :version
+    field_reader :author, :_csv
+    field_reader :maintainer
+    field_reader :sentence
+    field_reader :paragraph
+    field_reader :category
+    field_reader :url
+    field_reader :architectures, :_csv
+    field_reader :depends, :_csv
+    field_reader :dot_a_linkage, :_bool
+    field_reader :includes, :_csv
+    field_reader :precompiled, :_bool
+    field_reader :ldflags, :_csv
+
+    # The value of sentence always will be prepended, so you should start by writing the second sentence here
+    #
+    # (according to the docs)
+    # @return [String] the sentence and paragraph together
+    def full_paragraph
+      [sentence, paragraph].join(" ")
+    end
+
+  end
+
+end