arduino_ci 0.3.0 → 1.3.0

data/exe/arduino_ci_remote.rb

@@ -1,394 +1,3 @@
 #!/usr/bin/env ruby
-require 'arduino_ci'
-require 'set'
-require 'pathname'
-require 'optparse'
-
-WIDTH = 80
-FIND_FILES_INDENT = 4
-
-@failure_count = 0
-@passfail = proc { |result| result ? "✓" : "✗" }
-
-# Use some basic parsing to allow command-line overrides of config
-class Parser
-  def self.parse(options)
-    unit_config = {}
-    output_options = {
-      skip_unittests: false,
-      skip_compilation: false,
-      ci_config: {
-        "unittest" => unit_config
-      },
-    }
-
-    opt_parser = OptionParser.new do |opts|
-      opts.banner = "Usage: #{File.basename(__FILE__)} [options]"
-
-      opts.on("--skip-unittests", "Don't run unit tests") do |p|
-        output_options[:skip_unittests] = p
-      end
-
-      opts.on("--skip-compilation", "Don't compile example sketches") do |p|
-        output_options[:skip_compilation] = p
-      end
-
-      opts.on("--testfile-select=GLOB", "Unit test file (or glob) to select") do |p|
-        unit_config["testfiles"] ||= {}
-        unit_config["testfiles"]["select"] ||= []
-        unit_config["testfiles"]["select"] << p
-      end
-
-      opts.on("--testfile-reject=GLOB", "Unit test file (or glob) to reject") do |p|
-        unit_config["testfiles"] ||= {}
-        unit_config["testfiles"]["reject"] ||= []
-        unit_config["testfiles"]["reject"] << p
-      end
-
-      opts.on("-h", "--help", "Prints this help") do
-        puts opts
-        exit
-      end
-    end
-
-    opt_parser.parse!(options)
-    output_options
-  end
-end
-
-# Read in command line options and make them read-only
-@cli_options = (Parser.parse ARGV).freeze
-
-# terminate after printing any debug info.  TODO: capture debug info
-def terminate(final = nil)
-  puts "Failures: #{@failure_count}"
-  unless @failure_count.zero? || final
-    puts "Last message: #{@arduino_cmd.last_msg}"
-    puts "========== Stdout:"
-    puts @arduino_cmd.last_out
-    puts "========== Stderr:"
-    puts @arduino_cmd.last_err
-  end
-  retcode = @failure_count.zero? ? 0 : 1
-  exit(retcode)
-end
-
-# make a nice status line for an action and react to the action
-# TODO / note to self: inform_multline is tougher to write
-#   without altering the signature because it only leaves space
-#   for the checkmark _after_ the multiline, it doesn't know how
-#   to make that conditionally the body
-# @param message String the text of the progress indicator
-# @param multiline boolean whether multiline output is expected
-# @param mark_fn block (string) -> string that says how to describe the result
-# @param on_fail_msg String custom message for failure
-# @param tally_on_fail boolean whether to increment @failure_count
-# @param abort_on_fail boolean whether to abort immediately on failure (i.e. if this is a fatal error)
-def perform_action(message, multiline, mark_fn, on_fail_msg, tally_on_fail, abort_on_fail)
-  line = "#{message}... "
-  endline = "...#{message} "
-  if multiline
-    puts line
-  else
-    print line
-  end
-  STDOUT.flush
-  result = yield
-  mark = mark_fn.nil? ? "" : mark_fn.call(result)
-  # if multline, put checkmark at full width
-  print endline if multiline
-  puts mark.to_s.rjust(WIDTH - line.length, " ")
-  unless result
-    puts on_fail_msg unless on_fail_msg.nil?
-    @failure_count += 1 if tally_on_fail
-    # print out error messaging here if we've captured it
-    terminate if abort_on_fail
-  end
-  result
-end
-
-# Make a nice status for something that defers any failure code until script exit
-def attempt(message, &block)
-  perform_action(message, false, @passfail, nil, true, false, &block)
-end
-
-# Make a nice status for something that defers any failure code until script exit
-def attempt_multiline(message, &block)
-  perform_action(message, true, @passfail, nil, true, false, &block)
-end
-
-# Make a nice status for something that kills the script immediately on failure
-FAILED_ASSURANCE_MESSAGE = "This may indicate a problem with ArduinoCI, or your configuration".freeze
-def assure(message, &block)
-  perform_action(message, false, @passfail, FAILED_ASSURANCE_MESSAGE, true, true, &block)
-end
-
-def assure_multiline(message, &block)
-  perform_action(message, true, @passfail, FAILED_ASSURANCE_MESSAGE, true, true, &block)
-end
-
-def inform(message, &block)
-  perform_action(message, false, proc { |x| x }, nil, false, false, &block)
-end
-
-def inform_multiline(message, &block)
-  perform_action(message, true, nil, nil, false, false, &block)
-end
-
-# Assure that a platform exists and return its definition
-def assured_platform(purpose, name, config)
-  platform_definition = config.platform_definition(name)
-  assure("Requested #{purpose} platform '#{name}' is defined in 'platforms' YML") do
-    !platform_definition.nil?
-  end
-  platform_definition
-end
-
-# Return true if the file (or one of the dirs containing it) is hidden
-def file_is_hidden_somewhere?(path)
-  # this is clunkly but pre-2.2-ish ruby doesn't return ascend as an enumerator
-  path.ascend do |part|
-    return true if part.basename.to_s.start_with? "."
-  end
-  false
-end
-
-# print out some files
-def display_files(pathname)
-  # `find` doesn't follow symlinks, so we should instead
-  realpath = pathname.symlink? ? pathname.readlink : pathname
-
-  # suppress directories and dotfile-based things
-  all_files = realpath.find.select(&:file?)
-  non_hidden = all_files.reject { |path| file_is_hidden_somewhere?(path) }
-
-  # print files with an indent
-  margin = " " * FIND_FILES_INDENT
-  non_hidden.each { |p| puts "#{margin}#{p}" }
-end
-
-def install_arduino_library_dependencies(aux_libraries)
-  aux_libraries.each do |l|
-    if @arduino_cmd.library_present?(l)
-      inform("Using pre-existing library") { l.to_s }
-    else
-      assure("Installing aux library '#{l}'") { @arduino_cmd.install_library(l) }
-    end
-  end
-end
-
-def perform_unit_tests(file_config)
-  if @cli_options[:skip_unittests]
-    inform("Skipping unit tests") { "as requested via command line" }
-    return
-  end
-  config = file_config.with_override_config(@cli_options[:ci_config])
-  cpp_library = ArduinoCI::CppLibrary.new(Pathname.new("."),
-                                          @arduino_cmd.lib_dir,
-                                          config.exclude_dirs.map(&Pathname.method(:new)))
-
-  # check GCC
-  compilers = config.compilers_to_use
-  assure("The set of compilers (#{compilers.length}) isn't empty") { !compilers.empty? }
-  compilers.each do |gcc_binary|
-    attempt_multiline("Checking #{gcc_binary} version") do
-      version = cpp_library.gcc_version(gcc_binary)
-      next nil unless version
-
-      puts version.split("\n").map { |l| "    #{l}" }.join("\n")
-      version
-    end
-    inform("libasan availability for #{gcc_binary}") { cpp_library.libasan?(gcc_binary) }
-  end
-
-  # Ensure platforms exist for unit test, and save their info in all_platform_info keyed by name
-  all_platform_info = {}
-  config.platforms_to_unittest.each { |p| all_platform_info[p] = assured_platform("unittest", p, config) }
-
-  # iterate boards / tests
-  if !cpp_library.tests_dir.exist?
-    inform_multiline("Skipping unit tests; no tests dir at #{cpp_library.tests_dir}") do
-      puts "  In case that's an error, this is what was found in the library:"
-      display_files(cpp_library.tests_dir.parent)
-      true
-    end
-  elsif cpp_library.test_files.empty?
-    inform_multiline("Skipping unit tests; no test files were found in #{cpp_library.tests_dir}") do
-      puts "  In case that's an error, this is what was found in the tests directory:"
-      display_files(cpp_library.tests_dir)
-      true
-    end
-  elsif config.platforms_to_unittest.empty?
-    inform("Skipping unit tests") { "no platforms were requested" }
-  else
-    install_arduino_library_dependencies(config.aux_libraries_for_unittest)
-
-    config.platforms_to_unittest.each do |p|
-      config.allowable_unittest_files(cpp_library.test_files).each do |unittest_path|
-        unittest_name = unittest_path.basename.to_s
-        compilers.each do |gcc_binary|
-          attempt_multiline("Unit testing #{unittest_name} with #{gcc_binary}") do
-            exe = cpp_library.build_for_test_with_configuration(
-              unittest_path,
-              config.aux_libraries_for_unittest,
-              gcc_binary,
-              config.gcc_config(p)
-            )
-            puts
-            unless exe
-              puts "Last command: #{cpp_library.last_cmd}"
-              puts cpp_library.last_out
-              puts cpp_library.last_err
-              next false
-            end
-            cpp_library.run_test_file(exe)
-          end
-        end
-      end
-    end
-  end
-end
-
-def perform_compilation_tests(config)
-  if @cli_options[:skip_compilation]
-    inform("Skipping compilation of examples") { "as requested via command line" }
-    return
-  end
-
-  # index the existing libraries
-  attempt("Indexing libraries") { @arduino_cmd.index_libraries } unless @arduino_cmd.libraries_indexed
-
-  # initialize library under test
-  installed_library_path = attempt("Installing library under test") do
-    @arduino_cmd.install_local_library(Pathname.new("."))
-  end
-
-  if !installed_library_path.nil? && installed_library_path.exist?
-    inform("Library installed at") { installed_library_path.to_s }
-  else
-    assure_multiline("Library installed successfully") do
-      if installed_library_path.nil?
-        puts @arduino_cmd.last_msg
-      else
-        # print out the contents of the deepest directory we actually find
-        @arduino_cmd.lib_dir.ascend do |path_part|
-          next unless path_part.exist?
-
-          break display_files(path_part)
-        end
-        false
-      end
-    end
-  end
-  library_examples = @arduino_cmd.library_examples(installed_library_path)
-
-  # gather up all required boards for compilation so we can install them up front.
-  # start with the "platforms to unittest" and add the examples
-  # while we're doing that, get the aux libraries as well
-  example_platform_info = {}
-  board_package_url = {}
-  aux_libraries = Set.new(config.aux_libraries_for_build)
-  # while collecting the platforms, ensure they're defined
-
-  library_examples.each do |path|
-    ovr_config = config.from_example(path)
-    ovr_config.platforms_to_build.each do |platform|
-      # assure the platform if we haven't already
-      next if example_platform_info.key?(platform)
-
-      platform_info = assured_platform("library example", platform, config)
-      next if platform_info.nil?
-
-      example_platform_info[platform] = platform_info
-      package = platform_info[:package]
-      board_package_url[package] = ovr_config.package_url(package)
-    end
-    aux_libraries.merge(ovr_config.aux_libraries_for_build)
-  end
-
-  # with all platform info, we can extract unique packages and their urls
-  # do that, set the URLs, and download the packages
-  all_packages = example_platform_info.values.map { |v| v[:package] }.uniq.reject(&:nil?)
-
-  # inform about builtin packages
-  all_packages.select { |p| config.package_builtin?(p) }.each do |p|
-    inform("Using built-in board package") { p }
-  end
-
-  # make sure any non-builtin package has a URL defined
-  all_packages.reject { |p| config.package_builtin?(p) }.each do |p|
-    assure("Board package #{p} has a defined URL") { board_package_url[p] }
-  end
-
-  # set up all the board manager URLs.
-  # we can safely reject nils now, they would be for the builtins
-  all_urls = all_packages.map { |p| board_package_url[p] }.uniq.reject(&:nil?)
-
-  unless all_urls.empty?
-    assure("Setting board manager URLs") do
-      @arduino_cmd.board_manager_urls = all_urls
-    end
-  end
-
-  all_packages.each do |p|
-    assure("Installing board package #{p}") do
-      @arduino_cmd.install_boards(p)
-    end
-  end
-
-  install_arduino_library_dependencies(aux_libraries)
-
-  last_board = nil
-  if config.platforms_to_build.empty?
-    inform("Skipping builds") { "no platforms were requested" }
-    return
-  elsif library_examples.empty?
-    inform_multiline("Skipping builds; no examples found in #{installed_library_path}") do
-      display_files(installed_library_path)
-    end
-    return
-  end
-
-  attempt("Setting compiler warning level")  { @arduino_cmd.set_pref("compiler.warning_level", "all") }
-
-  # switching boards takes time, so iterate board first
-  # _then_ whichever examples match it
-  examples_by_platform = library_examples.each_with_object({}) do |example_path, acc|
-    ovr_config = config.from_example(example_path)
-    ovr_config.platforms_to_build.each do |p|
-      acc[p] = [] unless acc.key?(p)
-      acc[p] << example_path
-    end
-  end
-
-  examples_by_platform.each do |platform, example_paths|
-    board = example_platform_info[platform][:board]
-    assure("Switching to board for #{platform} (#{board})") { @arduino_cmd.use_board(board) } unless last_board == board
-    last_board = board
-
-    example_paths.each do |example_path|
-      example_name = File.basename(example_path)
-      attempt("Verifying #{example_name}") do
-        ret = @arduino_cmd.verify_sketch(example_path)
-        unless ret
-          puts
-          puts "Last command: #{@arduino_cmd.last_msg}"
-          puts @arduino_cmd.last_err
-        end
-        ret
-      end
-    end
-  end
-
-end
-
-# initialize command and config
-config = ArduinoCI::CIConfig.default.from_project_library
-
-@arduino_cmd = ArduinoCI::ArduinoInstallation.autolocate!
-inform("Located Arduino binary") { @arduino_cmd.binary_path.to_s }
-
-perform_unit_tests(config)
-perform_compilation_tests(config)
-
-terminate(true)
+puts "arduino_ci_remote.rb is deprecated in favor of arduino_ci.rb."
+require_relative "arduino_ci"

data/exe/arduino_library_location.rb

@@ -2,6 +2,6 @@
 require 'arduino_ci'
 
 # locate and/or forcibly install Arduino, keep stdout clean
-@arduino_cmd = ArduinoCI::ArduinoInstallation.autolocate!($stderr)
+@backend = ArduinoCI::ArduinoInstallation.autolocate!($stderr)
 
-puts @arduino_cmd.lib_dir
+puts @backend.lib_dir

data/exe/ensure_arduino_installation.rb

@@ -2,4 +2,10 @@
 require 'arduino_ci'
 
 # this will exit after Arduino is located and/or forcibly installed
-ArduinoCI::ArduinoInstallation.autolocate!
+backend = ArduinoCI::ArduinoInstallation.autolocate!
+lib_dir = backend.lib_dir
+
+unless lib_dir.exist?
+  puts "Creating libraries directory #{lib_dir}"
+  lib_dir.mkpath
+end

data/lib/arduino_ci.rb

@@ -2,6 +2,7 @@ require "arduino_ci/version"
 require "arduino_ci/arduino_installation"
 require "arduino_ci/cpp_library"
 require "arduino_ci/ci_config"
+require "arduino_ci/library_properties"
 
 # ArduinoCI contains classes for automated testing of Arduino code on the command line
 # @author Ian Katz <ianfixes@gmail.com>

data/lib/arduino_ci/arduino_backend.rb

@@ -0,0 +1,238 @@
+require 'fileutils'
+require 'pathname'
+require 'json'
+
+# workaround for https://github.com/arduino/Arduino/issues/3535
+WORKAROUND_LIB = "USBHost".freeze
+
+module ArduinoCI
+
+  # To report errors that we can't resolve or possibly even explain
+  class ArduinoExecutionError < StandardError; end
+
+  # Wrap the Arduino executable.  This requires, in some cases, a faked display.
+  class ArduinoBackend
+
+    # We never even use this in code, it's just here for reference because the backend is picky about it. Used for testing
+    # @return [String] the only allowable name for the arduino-cli config file.
+    CONFIG_FILE_NAME = "arduino-cli.yaml".freeze
+
+    # the actual path to the executable on this platform
+    # @return [Pathname]
+    attr_accessor :binary_path
+
+    # If a custom config is deired (i.e. for testing), specify it here.
+    # Note https://github.com/arduino/arduino-cli/issues/753 : the --config-file option
+    # is really the director that contains the file
+    # @return [Pathname]
+    attr_accessor :config_dir
+
+    # @return [String] STDOUT of the most recently-run command
+    attr_reader   :last_out
+
+    # @return [String] STDERR of the most recently-run command
+    attr_reader   :last_err
+
+    # @return [String] the most recently-run command
+    attr_reader   :last_msg
+
+    # @return [Array<String>] Additional URLs for the boards manager
+    attr_reader   :additional_urls
+
+    def initialize(binary_path)
+      @binary_path        = binary_path
+      @config_dir         = nil
+      @additional_urls    = []
+      @last_out           = ""
+      @last_err           = ""
+      @last_msg           = ""
+    end
+
+    def _wrap_run(work_fn, *args, **kwargs)
+      # do some work to extract & merge environment variables if they exist
+      has_env = !args.empty? && args[0].instance_of?(Hash)
+      env_vars = has_env ? args[0] : {}
+      actual_args = has_env ? args[1..-1] : args  # need to shift over if we extracted args
+      custom_config = @config_dir.nil? ? [] : ["--config-file", @config_dir.to_s]
+      full_args = [binary_path.to_s, "--format", "json"] + custom_config + actual_args
+      full_cmd = env_vars.empty? ? full_args : [env_vars] + full_args
+
+      shell_vars = env_vars.map { |k, v| "#{k}=#{v}" }.join(" ")
+      @last_msg = " $ #{shell_vars} #{full_args.join(' ')}"
+      work_fn.call(*full_cmd, **kwargs)
+    end
+
+    # build and run the arduino command
+    def run_and_output(*args, **kwargs)
+      _wrap_run((proc { |*a, **k| Host.run_and_output(*a, **k) }), *args, **kwargs)
+    end
+
+    # run a command and capture its output
+    # @return [Hash] {:out => String, :err => String, :success => bool}
+    def run_and_capture(*args, **kwargs)
+      ret = _wrap_run((proc { |*a, **k| Host.run_and_capture(*a, **k) }), *args, **kwargs)
+      @last_err = ret[:err]
+      @last_out = ret[:out]
+      ret
+    end
+
+    def capture_json(*args, **kwargs)
+      ret = run_and_capture(*args, **kwargs)
+      ret[:json] = JSON.parse(ret[:out])
+      ret
+    end
+
+    # Get a dump of the entire config
+    # @return [Hash] The configuration
+    def config_dump
+      capture_json("config", "dump")[:json]
+    end
+
+    # @return [String] the path to the Arduino libraries directory
+    def lib_dir
+      Pathname.new(config_dump["directories"]["user"]) + "libraries"
+    end
+
+    # Board manager URLs
+    # @return [Array<String>] The additional URLs used by the board manager
+    def board_manager_urls
+      config_dump["board_manager"]["additional_urls"] + @additional_urls
+    end
+
+    # Set board manager URLs
+    # @return [Array<String>] The additional URLs used by the board manager
+    def board_manager_urls=(all_urls)
+      raise ArgumentError("all_urls should be an array, got #{all_urls.class}") unless all_urls.is_a? Array
+
+      @additional_urls = all_urls
+    end
+
+    # check whether a board is installed
+    # we do this by just selecting a board.
+    #   the arduino binary will error if unrecognized and do a successful no-op if it's installed
+    # @param boardname [String] The board to test
+    # @return [bool] Whether the board is installed
+    def board_installed?(boardname)
+      run_and_capture("board", "details", "--fqbn", boardname)[:success]
+    end
+
+    # check whether a board family is installed (e.g. arduino:avr)
+    #
+    # @param boardfamily_name [String] The board family to test
+    # @return [bool] Whether the board is installed
+    def boards_installed?(boardfamily_name)
+      capture_json("core", "list")[:json].any? { |b| b["ID"] == boardfamily_name }
+    end
+
+    # install a board by name
+    # @param name [String] the board name
+    # @return [bool] whether the command succeeded
+    def install_boards(boardfamily)
+      result = if @additional_urls.empty?
+        run_and_capture("core", "install", boardfamily)
+      else
+        run_and_capture("core", "install", boardfamily, "--additional-urls", @additional_urls.join(","))
+      end
+      result[:success]
+    end
+
+    # Find out if a library is available
+    #
+    # @param name [String] the library name
+    # @return [bool] whether the library can be installed via the library manager
+    def library_available?(name)
+      # the --names flag limits the size of the response to just the name field
+      capture_json("lib", "search", "--names", name)[:json]["libraries"].any? { |l| l["name"] == name }
+    end
+
+    # @return [Hash] information about installed libraries via the CLI
+    def installed_libraries
+      capture_json("lib", "list")[:json]
+    end
+
+    # @param path [String] The sketch to compile
+    # @param boardname [String] The board to use
+    # @return [bool] whether the command succeeded
+    def compile_sketch(path, boardname)
+      ext = File.extname path
+      unless ext.casecmp(".ino").zero?
+        @last_msg = "Refusing to compile sketch with '#{ext}' extension -- rename it to '.ino'!"
+        return false
+      end
+      unless File.exist? path
+        @last_msg = "Can't compile Sketch at nonexistent path '#{path}'!"
+        return false
+      end
+      ret = run_and_capture("compile", "--fqbn", boardname, "--warnings", "all", "--dry-run", path.to_s)
+      ret[:success]
+    end
+
+    # Guess the name of a library
+    # @param path [Pathname] The path to the library (installed or not)
+    # @return [String] the probable library name
+    def name_of_library(path)
+      src_path = path.realpath
+      properties_file = src_path + CppLibrary::LIBRARY_PROPERTIES_FILE
+      return src_path.basename.to_s unless properties_file.exist?
+      return src_path.basename.to_s if LibraryProperties.new(properties_file).name.nil?
+
+      LibraryProperties.new(properties_file).name
+    end
+
+    # Create a handle to an Arduino library by name
+    # @param name [String] The library "real name"
+    # @return [CppLibrary] The library object
+    def library_of_name(name)
+      raise ArgumentError, "name is not a String (got #{name.class})" unless name.is_a? String
+
+      CppLibrary.new(name, self)
+    end
+
+    # Create a handle to an Arduino library by path
+    # @param path [Pathname] The path to the library
+    # @return [CppLibrary] The library object
+    def library_of_path(path)
+      # the path must exist... and if it does, brute-force search the installed libs for it
+      realpath = path.realpath  # should produce error if the path doesn't exist to begin with
+      entry = installed_libraries.find { |l| Pathname.new(l["library"]["install_dir"]).realpath == realpath }
+      probable_name = entry["real_name"].nil? ? realpath.basename.to_s : entry["real_name"]
+      CppLibrary.new(probable_name, self)
+    end
+
+    # install a library from a path on the local machine (not via library manager), by symlink or no-op as appropriate
+    # @param path [Pathname] library to use
+    # @return [CppLibrary] the installed library, or nil
+    def install_local_library(path)
+      src_path         = path.realpath
+      library_name     = name_of_library(path)
+      cpp_library      = library_of_name(library_name)
+      destination_path = cpp_library.path
+
+      # things get weird if the sketchbook contains the library.
+      # check that first
+      if cpp_library.installed?
+        # maybe the project has always lived in the libraries directory, no need to symlink
+        return cpp_library if destination_path == src_path
+
+        uhoh = "There is already a library '#{library_name}' in the library directory (#{destination_path})"
+        # maybe it's a symlink? that would be OK
+        if Host.symlink?(destination_path)
+          current_destination_target = Host.readlink(destination_path)
+          return cpp_library if current_destination_target == src_path
+
+          @last_msg = "#{uhoh} and it's symlinked to #{current_destination_target} (expected #{src_path})"
+          return nil
+        end
+
+        @last_msg = "#{uhoh}.  It may need to be removed manually."
+        return nil
+      end
+
+      # install the library
+      libraries_dir = destination_path.parent
+      libraries_dir.mkpath unless libraries_dir.exist?
+      Host.symlink(src_path, destination_path)
+      cpp_library
+    end
+  end
+end