arduino_ci 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0c01ddca13fd4bd941dcff9e9faeeae700f184bbf647876d520c7ccfce0c87b3
-  data.tar.gz: 974ce43ef7eba01113b9470750c78ad4cc5f81f0296f72f18ce43def9ad1c597
+  metadata.gz: cad9df4f8ade3fbe801da837c8b6a73d129c9a5497e1d43b4f655de8d9b5b640
+  data.tar.gz: b42f9624e6af593c467d0f67a94b0695fb36b584d740f4fc38e256b8f703c075
 SHA512:
-  metadata.gz: a044d9ab06badda8a648f189786c14209d7bee59d4ba299dc8790bd29e54adfd107f15a53fcb28d7ad7f286c6daf3e559f6a09b15b3f625c5344c400b0a4c5e8
-  data.tar.gz: f134b211ce71483ee88794a8a9d9f742deed82ee3d93ccecbbdd745ff53af12d6b44f71e2480c7ea98ce60d62bb28f0553c22b773bd6376ea7854bae29f19c9c
+  metadata.gz: 25d40817ac30d531979d197768aad671e8efa9b3e2bc023fde41f3dbcbbb1b829a4f6a9915714a6eec2e1e234f2f70749390802431af81a7b23cf8b847a39750
+  data.tar.gz: 9ef54c3d3fc6704ad70d347b85f2fc039d697a81edfafaa3e9fea329cc20c0a0919f42267452e4f62be5d508460f38a3264f04b0c51ea98a3b98062cc4bd18b1

data/README.md

@@ -1,32 +1,44 @@
 [![Gem Version](https://badge.fury.io/rb/arduino_ci.svg)](https://rubygems.org/gems/arduino_ci)
 [![Build Status](https://travis-ci.org/ifreecarve/arduino_ci.svg)](https://travis-ci.org/ifreecarve/arduino_ci)
-[![Documentation](http://img.shields.io/badge/docs-rdoc.info-blue.svg)](http://www.rubydoc.info/gems/arduino_ci/)
+[![Documentation](http://img.shields.io/badge/docs-rdoc.info-blue.svg)](http://www.rubydoc.info/gems/arduino_ci/0.1.0)
 
 # ArduinoCI Ruby gem (`arduino_ci`)
 
 [Arduino CI](https://github.com/ifreecarve/arduino_ci) is a Ruby gem for executing Continuous Integration (CI) tests on an Arduino library -- both locally and as part of a service like Travis CI.
 
 
-## Installation
+## Installation In Your GitHub Project And Using Travis CI
 
-Add this line to your application's Gemfile:
+Add a file called `Gemfile` (no extension) to your Arduino project:
 
 ```ruby
+source 'https://rubygems.org'
 gem 'arduino_ci'
 ```
 
-And then execute:
+Next, you need this in `.travis.yml`
 
-    $ bundle
+```yaml
+sudo: false
+language: ruby
+script:
+   - bundle install
+   - bundle exec arduino_ci_remote.rb
+```
+
+That's literally all there is to it on the repository side.  You'll need to go to https://travis-ci.org/profile/ and enable testing for your Arduino project.  Once that happens, you should be all set.
 
-Or install it yourself as:
 
-    $ gem install arduino_ci
+## More Documentation
 
+This software is in alpha.  But [SampleProjects/DoSomething](SampleProjects/DoSomething) has a decent writeup and is a good bare-bones example of all the features.
 
-## Usage
+## Known Problems
 
-TODO: Write usage instructions here, based on other TODO of writing the actual gem.
+* The Arduino library is not fully mocked.
+* I don't have preprocessor defines for all the Arduino board flavors
+* Arduino Zero boards don't work in CI.  I'm confused.
+* https://github.com/ifreecarve/arduino_ci/issues
 
 
 ## Author
@@ -37,3 +49,5 @@ This gem was written by Ian Katz (ifreecarve@gmail.com) in 2018.  It's released
 ## See Also
 
 * [Contributing](CONTRIBUTING.md)
+* [Adafruit/travis-ci-arduino](https://github.com/adafruit/travis-ci-arduino) which inspired this project
+* [mmurdoch/arduinounit](https://github.com/mmurdoch/arduinounit) from which the unit test macros were adopted

data/exe/arduino_ci_remote.rb

@@ -0,0 +1,132 @@
+#!/usr/bin/env ruby
+require 'arduino_ci'
+require 'set'
+
+WIDTH = 80
+
+@failure_count = 0
+
+# terminate after printing any debug info.  TODO: capture debug info
+def terminate(final = nil)
+  puts "Failures: #{@failure_count}"
+  unless @failure_count.zero? || final
+    puts "Last message: #{@arduino_cmd.last_msg}"
+    puts "========== Stdout:"
+    puts @arduino_cmd.last_out
+    puts "========== Stderr:"
+    puts @arduino_cmd.last_err
+  end
+  retcode = @failure_count.zero? ? 0 : 1
+  exit(retcode)
+end
+
+# make a nice status line for an action and react to the action
+def perform_action(message, on_fail_msg, abort_on_fail)
+  line = "#{message}..."
+  print line
+  result = yield
+  mark = result ? "✓" : "X"
+  puts mark.rjust(WIDTH - line.length, " ")
+  unless result
+    puts on_fail_msg unless on_fail_msg.nil?
+    @failure_count += 1
+    # print out error messaging here if we've captured it
+    terminate if abort_on_fail
+  end
+  result
+end
+
+# Make a nice status for something that defers any failure code until script exit
+def attempt(message, &block)
+  perform_action(message, nil, false, &block)
+end
+
+# Make a nice status for something that kills the script immediately on failure
+def assure(message, &block)
+  perform_action(message, "This may indicate a problem with ArduinoCI!", true, &block)
+end
+
+# initialize command and config
+config = ArduinoCI::CIConfig.default.from_project_library
+@arduino_cmd = ArduinoCI::ArduinoInstallation.autolocate!
+
+# initialize library under test
+installed_library_path = assure("Installing library under test") { @arduino_cmd.install_local_library(".") }
+library_examples = @arduino_cmd.library_examples(installed_library_path)
+cpp_library = ArduinoCI::CppLibrary.new(installed_library_path)
+attempt("Library installed at #{installed_library_path}") { true }
+
+# gather up all required boards so we can install them up front.
+# start with the "platforms to unittest" and add the examples
+# while we're doing that, get the aux libraries as well
+all_platforms = {}
+aux_libraries = Set.new(config.aux_libraries_for_unittest + config.aux_libraries_for_build)
+config.platforms_to_unittest.each { |p| all_platforms[p] = config.platform_definition(p) }
+library_examples.each do |path|
+  ovr_config = config.from_example(path)
+  ovr_config.platforms_to_build.each { |p| all_platforms[p] = config.platform_definition(p) }
+  aux_libraries.merge(ovr_config.aux_libraries_for_build)
+end
+
+# with all platform info, we can extract unique packages and their urls
+# do that, set the URLs, and download the packages
+all_packages = all_platforms.values.map { |v| v[:package] }.uniq.reject(&:nil?)
+all_urls = all_packages.map { |p| config.package_url(p) }.uniq.reject(&:nil?)
+assure("Setting board manager URLs") do
+  @arduino_cmd.set_pref("boardsmanager.additional.urls", all_urls.join(","))
+end
+
+all_packages.each do |p|
+  assure("Installing board package #{p}") do
+    @arduino_cmd.install_boards(p)
+  end
+end
+
+aux_libraries.each do |l|
+  assure("Installing aux library '#{l}'") { @arduino_cmd.install_library(l) }
+end
+
+attempt("Setting compiler warning level") { @arduino_cmd.set_pref("compiler.warning_level", "all") }
+
+library_examples.each do |example_path|
+  ovr_config = config.from_example(example_path)
+  ovr_config.platforms_to_build.each do |p|
+    board = all_platforms[p][:board]
+    assure("Switching to board for #{p} (#{board})") { @arduino_cmd.use_board(board) }
+    example_name = File.basename(example_path)
+    attempt("Verifying #{example_name}") do
+      ret = @arduino_cmd.verify_sketch(example_path)
+      unless ret
+        puts
+        puts "Last command: #{@arduino_cmd.last_msg}"
+        puts @arduino_cmd.last_err
+      end
+      ret
+    end
+  end
+end
+
+config.platforms_to_unittest.each do |p|
+  board = all_platforms[p][:board]
+  assure("Switching to board for #{p} (#{board})") { @arduino_cmd.use_board(board) }
+  cpp_library.test_files.each do |unittest_path|
+    unittest_name = File.basename(unittest_path)
+    attempt("Unit testing #{unittest_name}") do
+      exe = cpp_library.build_for_test_with_configuration(
+        unittest_path,
+        config.aux_libraries_for_unittest,
+        config.gcc_config(p)
+      )
+      puts
+      unless exe
+        puts "Last command: #{cpp_library.last_cmd}"
+        puts cpp_library.last_out
+        puts cpp_library.last_err
+        next false
+      end
+      cpp_library.run_test_file(exe)
+    end
+  end
+end
+
+terminate(true)

data/lib/arduino_ci.rb

@@ -1,124 +1,10 @@
 require "arduino_ci/version"
-
-require 'singleton'
-
-# Cross-platform way of finding an executable in the $PATH.
-# via https://stackoverflow.com/a/5471032/2063546
-#   which('ruby') #=> /usr/bin/ruby
-def which(cmd)
-  exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
-  ENV['PATH'].split(File::PATH_SEPARATOR).each do |path|
-    exts.each do |ext|
-      exe = File.join(path, "#{cmd}#{ext}")
-      return exe if File.executable?(exe) && !File.directory?(exe)
-    end
-  end
-  nil
-end
+require "arduino_ci/arduino_installation"
+require "arduino_ci/cpp_library"
+require "arduino_ci/ci_config"
 
 # ArduinoCI contains classes for automated testing of Arduino code on the command line
 # @author Ian Katz <ifreecarve@gmail.com>
 module ArduinoCI
 
-  # Wrap the Arduino executable.  This requires, in some cases, a faked display.
-  class ArduinoCmd
-
-    # create as many ArduinoCmds as you like, but we need one and only one display manager
-    class DisplayMgr
-      include Singleton
-      attr_reader :enabled
-
-      def initialize
-        @existing = existing_display?
-        @enabled = false
-        @pid = nil
-      end
-
-      # attempt to determine if the machine is running a graphical display (i.e. not Travis)
-      def existing_display?
-        return true if RUBY_PLATFORM.include? "darwin"
-        return true if ENV["DISPLAY"].nil?
-        return true if ENV["DISPLAY"].include? ":"
-        false
-      end
-
-      # enable a virtual display
-      def enable
-        return @enabled = true if @existing   # silent no-op if built in display
-        return unless @pid.nil?
-
-        @enabled = true
-        @pid = fork do
-          puts "Forking Xvfb"
-          system("Xvfb", ":1", "-ac", "-screen", "0", "1280x1024x16")
-          puts "Xvfb unexpectedly quit!"
-        end
-        sleep(3)  # TODO: test a connection to the X server?
-      end
-
-      # disable the virtual display
-      def disable
-        return @enabled = false if @existing  # silent no-op if built in display
-        return if @pid.nil?
-
-        begin
-          Process.kill 9, @pid
-        ensure
-          Process.wait @pid
-          @pid = nil
-        end
-        puts "Xvfb killed"
-      end
-
-      # Enable a virtual display for the duration of the given block
-      def with_display
-        enable
-        begin
-          yield environment
-        ensure
-          disable
-        end
-      end
-
-      def environment
-        return nil unless @existing || @enabled
-        return {} if @existing
-        { DISPLAY => ":1.0" }
-      end
-
-      # On finalize, ensure child process is ended
-      def self.finalize
-        disable
-      end
-    end
-
-    class << self
-      protected :new
-
-      # attempt to find a workable Arduino executable across platforms
-      def guess_executable_location
-        osx_place = "/Applications/Arduino.app/Contents/MacOS/Arduino"
-        places = {
-          "arduino" => !which("arduino").nil?,
-          osx_place => (File.exist? osx_place),
-        }
-        places.each { |k, v| return k if v }
-        nil
-      end
-
-      def autolocate
-        ret = new
-        ret.path = guess_executable_location
-        ret
-      end
-    end
-
-    attr_accessor :path
-
-    def initialize
-      @display_mgr = DisplayMgr::instance
-    end
-
-  end
-
 end

data/lib/arduino_ci/arduino_cmd.rb

@@ -0,0 +1,296 @@
+require 'fileutils'
+
+module ArduinoCI
+
+  # Wrap the Arduino executable.  This requires, in some cases, a faked display.
+  class ArduinoCmd
+
+    # Enable a shortcut syntax for command line flags
+    # @param name [String] What the flag will be called (prefixed with 'flag_')
+    # @return [void]
+    # @macro [attach] flag
+    #   The text of the command line flag for $1
+    #   @!attribute [r] flag_$1
+    #   @return [String] the text of the command line flag (`$2` in this case)
+    def self.flag(name, text = nil)
+      text = "(flag #{name} not defined)" if text.nil?
+      self.class_eval("def flag_#{name};\"#{text}\";end")
+    end
+
+    # the path to the Arduino executable
+    # @return [String]
+    attr_accessor :base_cmd
+
+    # part of a workaround for https://github.com/arduino/Arduino/issues/3535
+    attr_reader   :library_is_indexed
+
+    # @return [String] STDOUT of the most recently-run command
+    attr_reader   :last_out
+
+    # @return [String] STDERR of the most recently-run command
+    attr_reader   :last_err
+
+    # @return [String] the most recently-run command
+    attr_reader   :last_msg
+
+    # set the command line flags (undefined for now).
+    # These vary between gui/cli
+    flag :get_pref
+    flag :set_pref
+    flag :save_prefs
+    flag :use_board
+    flag :install_boards
+    flag :install_library
+    flag :verify
+
+    def initialize
+      @prefs_cache        = {}
+      @prefs_fetched      = false
+      @library_is_indexed = false
+      @last_out           = ""
+      @last_err           = ""
+      @last_msg           = ""
+    end
+
+    # Convert a preferences dump into a flat hash
+    # @param arduino_output [String] The raw Arduino executable output
+    # @return [Hash] preferences as a hash
+    def parse_pref_string(arduino_output)
+      lines = arduino_output.split("\n").select { |l| l.include? "=" }
+      ret = lines.each_with_object({}) do |e, acc|
+        parts = e.split("=", 2)
+        acc[parts[0]] = parts[1]
+        acc
+      end
+      ret
+    end
+
+    # @return [String] the path to the Arduino libraries directory
+    def _lib_dir
+      "<lib dir not defined>"
+    end
+
+    # fetch preferences in their raw form
+    # @return [String] Preferences as a set of lines
+    def _prefs_raw
+      resp = run_and_capture(flag_get_pref)
+      return nil unless resp[:success]
+      resp[:out]
+    end
+
+    # Get the Arduino preferences, from cache if possible
+    # @return [Hash] The full set of preferences
+    def prefs
+      prefs_raw = _prefs_raw unless @prefs_fetched
+      return nil if prefs_raw.nil?
+      @prefs_cache = parse_pref_string(prefs_raw)
+      @prefs_cache.clone
+    end
+
+    # get a preference key
+    # @param key [String] The preferences key to look up
+    # @return [String] The preference value
+    def get_pref(key)
+      data = @prefs_fetched ? @prefs_cache : prefs
+      data[key]
+    end
+
+    # underlying preference-setter.
+    # @param key [String] The preference name
+    # @param value [String] The value to set to
+    # @return [bool] whether the command succeeded
+    def _set_pref(key, value)
+      run_and_capture(flag_set_pref, "#{key}=#{value}", flag_save_prefs)[:success]
+    end
+
+    # set a preference key/value pair, and update the cache.
+    # @param key [String] the preference key
+    # @param value [String] the preference value
+    # @return [bool] whether the command succeeded
+    def set_pref(key, value)
+      success = _set_pref(key, value)
+      @prefs_cache[key] = value if success
+      success
+    end
+
+    # run the arduino command
+    def _run(*args, **kwargs)
+      raise "Ian needs to implement this in a subclass #{args} #{kwargs}"
+    end
+
+    # build and run the arduino command
+    def run(*args, **kwargs)
+      # do some work to extract & merge environment variables if they exist
+      has_env = !args.empty? && args[0].class == Hash
+      env_vars = has_env ? args[0] : {}
+      actual_args = has_env ? args[1..-1] : args  # need to shift over if we extracted args
+      full_args = @base_cmd + actual_args
+      full_cmd = env_vars.empty? ? full_args : [env_vars] + full_args
+
+      shell_vars = env_vars.map { |k, v| "#{k}=#{v}" }.join(" ")
+      @last_msg = " $ #{shell_vars} #{full_args.join(' ')}"
+      _run(*full_cmd, **kwargs)
+    end
+
+    # run a command and capture its output
+    # @return [Hash] {:out => String, :err => String, :success => bool}
+    def run_and_capture(*args, **kwargs)
+      pipe_out, pipe_out_wr = IO.pipe
+      pipe_err, pipe_err_wr = IO.pipe
+      our_kwargs = { out: pipe_out_wr, err: pipe_err_wr }
+      eventual_kwargs = our_kwargs.merge(kwargs)
+      success = run(*args, **eventual_kwargs)
+      pipe_out_wr.close
+      pipe_err_wr.close
+      str_out = pipe_out.read
+      str_err = pipe_err.read
+      pipe_out.close
+      pipe_err.close
+      @last_err = str_err
+      @last_out = str_out
+      { out: str_out, err: str_err, success: success }
+    end
+
+    # run a command and don't capture its output, but use the same signature
+    # @return [Hash] {:out => String, :err => String, :success => bool}
+    def run_wrap(*args, **kwargs)
+      success = run(*args, **kwargs)
+      { out: "NOPE, use run_and_capture", err: "NOPE, use run_and_capture", success: success }
+    end
+
+    # check whether a board is installed
+    # we do this by just selecting a board.
+    #   the arduino binary will error if unrecognized and do a successful no-op if it's installed
+    # @param boardname [String] The board to test
+    # @return [bool] Whether the board is installed
+    def board_installed?(boardname)
+      run_and_capture(flag_use_board, boardname)[:success]
+    end
+
+    # install a board by name
+    # @param name [String] the board name
+    # @return [bool] whether the command succeeded
+    def install_boards(boardfamily)
+      # TODO: find out why IO.pipe fails but File::NULL succeeds :(
+      result = run_and_capture(flag_install_boards, boardfamily)
+      already_installed = result[:err].include?("Platform is already installed!")
+      result[:success] || already_installed
+    end
+
+    # install a library by name
+    # @param name [String] the library name
+    # @return [bool] whether the command succeeded
+    def install_library(library_name)
+      # workaround for https://github.com/arduino/Arduino/issues/3535
+      # use a dummy library name but keep open the possiblity that said library
+      # might be selected by choice for installation
+      workaround_lib = "USBHost"
+      unless @library_is_indexed || workaround_lib == library_name
+        @library_is_indexed = run_and_capture(flag_install_library, workaround_lib)
+      end
+
+      # actual installation
+      result = run_and_capture(flag_install_library, library_name)
+
+      # update flag if necessary
+      @library_is_indexed = (@library_is_indexed || result[:success]) if library_name == workaround_lib
+      result[:success]
+    end
+
+    # generate the (very likely) path of a library given its name
+    # @param library_name [String] The name of the library
+    # @return [String] The fully qualified library name
+    def library_path(library_name)
+      File.join(_lib_dir, library_name)
+    end
+
+    # update the library index
+    # @return [bool] Whether the update succeeded
+    def update_library_index
+      # install random lib so the arduino IDE grabs a new library index
+      # see: https://github.com/arduino/Arduino/issues/3535
+      install_library("USBHost")
+    end
+
+    # use a particular board for compilation
+    # @param boardname [String] The board to use
+    # @return [bool] whether the command succeeded
+    def use_board(boardname)
+      run_and_capture(flag_use_board, boardname, flag_save_prefs)[:success]
+    end
+
+    # use a particular board for compilation, installing it if necessary
+    # @param boardname [String] The board to use
+    # @return [bool] whether the command succeeded
+    def use_board!(boardname)
+      return true if use_board(boardname)
+      boardfamily = boardname.split(":")[0..1].join(":")
+      puts "Board '#{boardname}' not found; attempting to install '#{boardfamily}'"
+      return false unless install_boards(boardfamily) # guess board family from first 2 :-separated fields
+      use_board(boardname)
+    end
+
+    # @param path [String] The sketch to verify
+    # @return [bool] whether the command succeeded
+    def verify_sketch(path)
+      ext = File.extname path
+      unless ext.casecmp(".ino").zero?
+        @last_msg = "Refusing to verify sketch with '#{ext}' extension -- rename it to '.ino'!"
+        return false
+      end
+      unless File.exist? path
+        @last_msg = "Can't verify Sketch at nonexistent path '#{path}'!"
+        return false
+      end
+      ret = run_and_capture(flag_verify, path)
+      puts "=============="
+      puts ret[:out]
+      puts "--------------"
+      puts ret[:err]
+      ret[:success]
+    end
+
+    # ensure that the given library is installed, or symlinked as appropriate
+    # return the path of the prepared library, or nil
+    # @param path [String] library to use
+    # @return [String] the path of the installed library
+    def install_local_library(path)
+      realpath = File.expand_path(path)
+      library_name = File.basename(realpath)
+      destination_path = library_path(library_name)
+
+      # things get weird if the sketchbook contains the library.
+      # check that first
+      if File.exist? destination_path
+        uhoh = "There is already a library '#{library_name}' in the library directory"
+        return destination_path if destination_path == realpath
+
+        # maybe it's a symlink? that would be OK
+        if File.symlink?(destination_path)
+          return destination_path if File.readlink(destination_path) == realpath
+          @last_msg = "#{uhoh} and it's not symlinked to #{realpath}"
+          return nil
+        end
+
+        @last_msg = "#{uhoh}.  It may need to be removed manually."
+        return nil
+      end
+
+      # install the library
+      FileUtils.ln_s(realpath, destination_path)
+      destination_path
+    end
+
+    # @param installed_library_path [String] The library to query
+    # @return [Array<String>] Example sketch files
+    def library_examples(installed_library_path)
+      example_path = File.join(installed_library_path, "examples")
+      examples = Pathname.new(example_path).children.select(&:directory?).map(&:to_path).map(&File.method(:basename))
+      files = examples.map do |e|
+        proj_file = File.join(example_path, e, "#{e}.ino")
+        File.exist?(proj_file) ? proj_file : nil
+      end
+      files.reject(&:nil?)
+    end
+  end
+end