arduino_ci 0.4.0 → 1.0.0

data/lib/arduino_ci/host.rb

@@ -6,6 +6,13 @@ module ArduinoCI
 
   # Tools for interacting with the host machine
   class Host
+    # TODO: this came from https://stackoverflow.com/a/22716582/2063546
+    #       and I'm not sure if it can be replaced by self.os == :windows
+    WINDOWS_VARIANT_REGEX = /mswin32|cygwin|mingw|bccwin/
+
+    # e.g. 11/27/2020  01:02 AM    <SYMLINKD>     ExcludeSomething [C:\projects\arduino-ci\SampleProjects\ExcludeSomething]
+    DIR_SYMLINK_REGEX = %r{\d+/\d+/\d+\s+[^<]+<SYMLINKD?>\s+(.*) \[([^\]]+)\]}
+
     # Cross-platform way of finding an executable in the $PATH.
     # via https://stackoverflow.com/a/5471032/2063546
     #   which('ruby') #=> /usr/bin/ruby
@@ -38,21 +45,69 @@ module ArduinoCI
       return :windows if OS.windows?
     end
 
+    # Cross-platform symlinking
     # if on windows, call mklink, else self.symlink
     # @param [Pathname] old_path
     # @param [Pathname] new_path
     def self.symlink(old_path, new_path)
-      return FileUtils.ln_s(old_path.to_s, new_path.to_s) unless RUBY_PLATFORM =~ /mswin32|cygwin|mingw|bccwin/
+      # we would prefer `new_path.make_symlink(old_path)` but "symlink function is unimplemented on this machine" with windows
+      return new_path.make_symlink(old_path) unless needs_symlink_hack?
 
-      # https://stackoverflow.com/a/22716582/2063546
+      # via https://stackoverflow.com/a/22716582/2063546
       # windows mklink syntax is reverse of unix ln -s
       # windows mklink is built into cmd.exe
       # vulnerable to command injection, but okay because this is a hack to make a cli tool work.
-      orp = old_path.realpath.to_s.tr("/", "\\") # HACK DUE TO REALPATH BUG where it
-      np = new_path.to_s.tr("/", "\\")           # still joins windows paths with '/'
+      orp = pathname_to_windows(old_path.realpath)
+      np  = pathname_to_windows(new_path)
 
       _stdout, _stderr, exitstatus = Open3.capture3('cmd.exe', "/C mklink /D #{np} #{orp}")
       exitstatus.success?
     end
+
+    # Hack for "realpath" which on windows joins paths with slashes instead of backslashes
+    # @param path [Pathname] the path to render
+    # @return [String] A path that will work on windows
+    def self.pathname_to_windows(path)
+      path.to_s.tr("/", "\\")
+    end
+
+    # Hack for "realpath" which on windows joins paths with slashes instead of backslashes
+    # @param str [String] the windows path
+    # @return [Pathname] A path that will be recognized by pathname
+    def self.windows_to_pathname(str)
+      Pathname.new(str.tr("\\", "/"))
+    end
+
+    # Whether this OS requires a hack for symlinks
+    # @return [bool]
+    def self.needs_symlink_hack?
+      RUBY_PLATFORM =~ WINDOWS_VARIANT_REGEX
+    end
+
+    # Cross-platform is-this-a-symlink function
+    # @param [Pathname] path
+    # @return [bool] Whether the file is a symlink
+    def self.symlink?(path)
+      return path.symlink? unless needs_symlink_hack?
+
+      !readlink(path).nil?
+    end
+
+    # Cross-platform "read link" function
+    # @param [Pathname] path
+    # @return [Pathname] the link target
+    def self.readlink(path)
+      return path.readlink unless needs_symlink_hack?
+
+      the_dir  = pathname_to_windows(path.parent)
+      the_file = path.basename.to_s
+
+      stdout, _stderr, _exitstatus = Open3.capture3('cmd.exe', "/c dir /al #{the_dir}")
+      symlinks = stdout.lines.map { |l| DIR_SYMLINK_REGEX.match(l) }.compact
+      our_link = symlinks.find { |m| m[1] == the_file }
+      return nil if our_link.nil?
+
+      windows_to_pathname(our_link[2])
+    end
   end
 end

data/lib/arduino_ci/library_properties.rb

@@ -11,12 +11,22 @@ module ArduinoCI
     # @param path [Pathname] The path to the library.properties file
     def initialize(path)
       @fields = {}
-      File.foreach(path) do |line|
+      File.foreach(path) do |line_with_delim|
+        line = line_with_delim.chomp
         parts = line.split("=", 2)
-        @fields[parts[0]] = parts[1].chomp unless parts.empty?
+        next if parts[0].nil?
+        next if parts[0].empty?
+        next if parts[1].nil?
+
+        @fields[parts[0]] = parts[1] unless parts[1].empty?
       end
     end
 
+    # @return [Hash] the properties as a hash, all strings
+    def to_h
+      @fields.clone
+    end
+
     # Enable a shortcut syntax for library property accessors, in the style of `attr_accessor` metaprogramming.
     # This is used to create a named field pointing to a specific property in the file, optionally applying
     # a specific formatting function.

data/lib/arduino_ci/version.rb

@@ -1,3 +1,3 @@
 module ArduinoCI
-  VERSION = "0.4.0".freeze
+  VERSION = "1.0.0".freeze
 end

data/misc/default.yml

@@ -3,8 +3,12 @@
 #   https://en.wikipedia.org/wiki/List_of_Arduino_boards_and_compatible_systems
 
 packages:
-  # arduino:xxx are builtin, we don't need to include them here
-  # but if we did, it would be url: https://downloads.arduino.cc/packages/package_index.json
+  arduino:avr:
+    url: https://downloads.arduino.cc/packages/package_index.json
+  arduino:sam:
+    url: https://downloads.arduino.cc/packages/package_index.json
+  arduino:samd:
+    url: https://downloads.arduino.cc/packages/package_index.json
   esp8266:esp8266:
     url: http://arduino.esp8266.com/stable/package_esp8266com_index.json
   adafruit:avr:

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: arduino_ci
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Ian Katz
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-11-22 00:00:00.000000000 Z
+date: 2020-11-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: os
@@ -38,76 +38,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">"
-      - !ruby/object:Gem::Version
-        version: '1.15'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">"
-      - !ruby/object:Gem::Version
-        version: '1.15'
-- !ruby/object:Gem::Dependency
-  name: keepachangelog_manager
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.0.2
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.0.2
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.59.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.59.0
-- !ruby/object:Gem::Dependency
-  name: yard
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.9.11
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.9.11
 description: ''
 email:
 - arduino.continuous.integration@gmail.com
@@ -437,11 +367,7 @@ files:
 - exe/arduino_library_location.rb
 - exe/ensure_arduino_installation.rb
 - lib/arduino_ci.rb
-- lib/arduino_ci/arduino_cmd.rb
-- lib/arduino_ci/arduino_cmd_linux.rb
-- lib/arduino_ci/arduino_cmd_linux_builder.rb
-- lib/arduino_ci/arduino_cmd_osx.rb
-- lib/arduino_ci/arduino_cmd_windows.rb
+- lib/arduino_ci/arduino_backend.rb
 - lib/arduino_ci/arduino_downloader.rb
 - lib/arduino_ci/arduino_downloader_linux.rb
 - lib/arduino_ci/arduino_downloader_osx.rb
@@ -450,7 +376,6 @@ files:
 - lib/arduino_ci/ci_config.rb
 - lib/arduino_ci/cpp_library.rb
 - lib/arduino_ci/host.rb
-- lib/arduino_ci/installed_cpp_library.rb
 - lib/arduino_ci/library_properties.rb
 - lib/arduino_ci/version.rb
 - misc/default.yml

data/lib/arduino_ci/arduino_cmd.rb

@@ -1,332 +0,0 @@
-require 'fileutils'
-require 'pathname'
-
-# workaround for https://github.com/arduino/Arduino/issues/3535
-WORKAROUND_LIB = "USBHost".freeze
-
-module ArduinoCI
-
-  # To report errors that we can't resolve or possibly even explain
-  class ArduinoExecutionError < StandardError; end
-
-  # Wrap the Arduino executable.  This requires, in some cases, a faked display.
-  class ArduinoCmd
-
-    # Enable a shortcut syntax for command line flags
-    # @param name [String] What the flag will be called (prefixed with 'flag_')
-    # @return [void]
-    # @macro [attach] flag
-    #   The text of the command line flag for $1
-    #   @!attribute [r] flag_$1
-    #   @return [String] the text of the command line flag (`$2` in this case)
-    def self.flag(name, text = nil)
-      text = "(flag #{name} not defined)" if text.nil?
-      self.class_eval("def flag_#{name};\"#{text}\";end", __FILE__, __LINE__)
-    end
-
-    # the array of command components to launch the Arduino executable
-    # @return [Array<String>]
-    attr_accessor :base_cmd
-
-    # the actual path to the executable on this platform
-    # @return [Pathname]
-    attr_accessor :binary_path
-
-    # part of a workaround for https://github.com/arduino/Arduino/issues/3535
-    attr_reader   :libraries_indexed
-
-    # @return [String] STDOUT of the most recently-run command
-    attr_reader   :last_out
-
-    # @return [String] STDERR of the most recently-run command
-    attr_reader   :last_err
-
-    # @return [String] the most recently-run command
-    attr_reader   :last_msg
-
-    # set the command line flags (undefined for now).
-    # These vary between gui/cli.  Inline comments added for greppability
-    flag :get_pref           # flag_get_pref
-    flag :set_pref           # flag_set_pref
-    flag :save_prefs         # flag_save_prefs
-    flag :use_board          # flag_use_board
-    flag :install_boards     # flag_install_boards
-    flag :install_library    # flag_install_library
-    flag :verify             # flag_verify
-
-    def initialize
-      @prefs_cache        = {}
-      @prefs_fetched      = false
-      @libraries_indexed  = false
-      @last_out           = ""
-      @last_err           = ""
-      @last_msg           = ""
-    end
-
-    # Convert a preferences dump into a flat hash
-    # @param arduino_output [String] The raw Arduino executable output
-    # @return [Hash] preferences as a hash
-    def parse_pref_string(arduino_output)
-      lines = arduino_output.split("\n").select { |l| l.include? "=" }
-      ret = lines.each_with_object({}) do |e, acc|
-        parts = e.split("=", 2)
-        acc[parts[0]] = parts[1]
-        acc
-      end
-      ret
-    end
-
-    # @return [String] the path to the Arduino libraries directory
-    def lib_dir
-      Pathname.new(get_pref("sketchbook.path")) + "libraries"
-    end
-
-    # fetch preferences in their raw form
-    # @return [String] Preferences as a set of lines
-    def _prefs_raw
-      resp = run_and_capture(flag_get_pref)
-      fail_msg = "Arduino binary failed to operate as expected; you will have to troubleshoot it manually"
-      raise ArduinoExecutionError, "#{fail_msg}. The command was #{@last_msg}" unless resp[:success]
-
-      @prefs_fetched = true
-      resp[:out]
-    end
-
-    # Get the Arduino preferences, from cache if possible
-    # @return [Hash] The full set of preferences
-    def prefs
-      prefs_raw = _prefs_raw unless @prefs_fetched
-      return nil if prefs_raw.nil?
-
-      @prefs_cache = parse_pref_string(prefs_raw)
-      @prefs_cache.clone
-    end
-
-    # get a preference key
-    # @param key [String] The preferences key to look up
-    # @return [String] The preference value
-    def get_pref(key)
-      data = @prefs_fetched ? @prefs_cache : prefs
-      data[key]
-    end
-
-    # underlying preference-setter.
-    # @param key [String] The preference name
-    # @param value [String] The value to set to
-    # @return [bool] whether the command succeeded
-    def _set_pref(key, value)
-      run_and_capture(flag_set_pref, "#{key}=#{value}", flag_save_prefs)[:success]
-    end
-
-    # set a preference key/value pair, and update the cache.
-    # @param key [String] the preference key
-    # @param value [String] the preference value
-    # @return [bool] whether the command succeeded
-    def set_pref(key, value)
-      prefs unless @prefs_fetched  # update cache first
-      success = _set_pref(key, value)
-      @prefs_cache[key] = value if success
-      success
-    end
-
-    def _wrap_run(work_fn, *args, **kwargs)
-      # do some work to extract & merge environment variables if they exist
-      has_env = !args.empty? && args[0].class == Hash
-      env_vars = has_env ? args[0] : {}
-      actual_args = has_env ? args[1..-1] : args  # need to shift over if we extracted args
-      full_args = @base_cmd + actual_args
-      full_cmd = env_vars.empty? ? full_args : [env_vars] + full_args
-
-      shell_vars = env_vars.map { |k, v| "#{k}=#{v}" }.join(" ")
-      @last_msg = " $ #{shell_vars} #{full_args.join(' ')}"
-      work_fn.call(*full_cmd, **kwargs)
-    end
-
-    # build and run the arduino command
-    def run_and_output(*args, **kwargs)
-      _wrap_run((proc { |*a, **k| Host.run_and_output(*a, **k) }), *args, **kwargs)
-    end
-
-    # run a command and capture its output
-    # @return [Hash] {:out => String, :err => String, :success => bool}
-    def run_and_capture(*args, **kwargs)
-      ret = _wrap_run((proc { |*a, **k| Host.run_and_capture(*a, **k) }), *args, **kwargs)
-      @last_err = ret[:err]
-      @last_out = ret[:out]
-      ret
-    end
-
-    # Board manager URLs
-    # @return [Array<String>] The additional URLs used by the board manager
-    def board_manager_urls
-      url_list = get_pref("boardsmanager.additional.urls")
-      return [] if url_list.nil?
-
-      url_list.split(",")
-    end
-
-    # Set board manager URLs
-    # @return [Array<String>] The additional URLs used by the board manager
-    def board_manager_urls=(all_urls)
-      set_pref("boardsmanager.additional.urls", all_urls.join(","))
-    end
-
-    # check whether a board is installed
-    # we do this by just selecting a board.
-    #   the arduino binary will error if unrecognized and do a successful no-op if it's installed
-    # @param boardname [String] The board to test
-    # @return [bool] Whether the board is installed
-    def board_installed?(boardname)
-      run_and_capture(flag_use_board, boardname)[:success]
-    end
-
-    # install a board by name
-    # @param name [String] the board name
-    # @return [bool] whether the command succeeded
-    def install_boards(boardfamily)
-      # TODO: find out why IO.pipe fails but File::NULL succeeds :(
-      result = run_and_capture(flag_install_boards, boardfamily)
-      already_installed = result[:err].include?("Platform is already installed!")
-      result[:success] || already_installed
-    end
-
-    # install a library by name
-    # @param name [String] the library name
-    # @return [bool] whether the command succeeded
-    def _install_library(library_name)
-      result = run_and_capture(flag_install_library, library_name)
-
-      already_installed = result[:err].include?("Library is already installed: #{library_name}")
-      success = result[:success] || already_installed
-
-      @libraries_indexed = (@libraries_indexed || success) if library_name == WORKAROUND_LIB
-      success
-    end
-
-    # index the set of libraries by installing a dummy library
-    # related to WORKAROUND_LIB and https://github.com/arduino/Arduino/issues/3535
-    # TODO: unclear if this is still necessary
-    def index_libraries
-      return true if @libraries_indexed
-
-      _install_library(WORKAROUND_LIB)
-      @libraries_indexed
-    end
-
-    # install a library by name
-    # @param name [String] the library name
-    # @return [bool] whether the command succeeded
-    def install_library(library_name)
-      index_libraries
-      _install_library(library_name)
-    end
-
-    # generate the (very likely) path of a library given its name
-    # @param library_name [String] The name of the library
-    # @return [Pathname] The fully qualified library name
-    def library_path(library_name)
-      Pathname.new(lib_dir) + library_name
-    end
-
-    # Determine whether a library is present in the lib dir
-    #
-    # Note that `true` doesn't guarantee that the library is valid/installed
-    #  and `false` doesn't guarantee that the library isn't built-in
-    #
-    # @param library_name [String] The name of the library
-    # @return [bool]
-    def library_present?(library_name)
-      library_path(library_name).exist?
-    end
-
-    # update the library index
-    # @return [bool] Whether the update succeeded
-    def update_library_index
-      # install random lib so the arduino IDE grabs a new library index
-      # see: https://github.com/arduino/Arduino/issues/3535
-      install_library(WORKAROUND_LIB)
-    end
-
-    # use a particular board for compilation
-    # @param boardname [String] The board to use
-    # @return [bool] whether the command succeeded
-    def use_board(boardname)
-      run_and_capture(flag_use_board, boardname, flag_save_prefs)[:success]
-    end
-
-    # use a particular board for compilation, installing it if necessary
-    # @param boardname [String] The board to use
-    # @return [bool] whether the command succeeded
-    def use_board!(boardname)
-      return true if use_board(boardname)
-
-      boardfamily = boardname.split(":")[0..1].join(":")
-      puts "Board '#{boardname}' not found; attempting to install '#{boardfamily}'"
-      return false unless install_boards(boardfamily) # guess board family from first 2 :-separated fields
-
-      use_board(boardname)
-    end
-
-    # @param path [String] The sketch to verify
-    # @return [bool] whether the command succeeded
-    def verify_sketch(path)
-      ext = File.extname path
-      unless ext.casecmp(".ino").zero?
-        @last_msg = "Refusing to verify sketch with '#{ext}' extension -- rename it to '.ino'!"
-        return false
-      end
-      unless File.exist? path
-        @last_msg = "Can't verify Sketch at nonexistent path '#{path}'!"
-        return false
-      end
-      ret = run_and_capture(flag_verify, path)
-      ret[:success]
-    end
-
-    # ensure that the given library is installed, or symlinked as appropriate
-    # return the path of the prepared library, or nil
-    # @param path [Pathname] library to use
-    # @return [String] the path of the installed library
-    def install_local_library(path)
-      src_path = path.realpath
-      library_name = src_path.basename
-      destination_path = library_path(library_name)
-
-      # things get weird if the sketchbook contains the library.
-      # check that first
-      if destination_path.exist?
-        uhoh = "There is already a library '#{library_name}' in the library directory"
-        return destination_path if destination_path == src_path
-
-        # maybe it's a symlink? that would be OK
-        if destination_path.symlink?
-          return destination_path if destination_path.readlink == src_path
-
-          @last_msg = "#{uhoh} and it's not symlinked to #{src_path}"
-          return nil
-        end
-
-        @last_msg = "#{uhoh}.  It may need to be removed manually."
-        return nil
-      end
-
-      # install the library
-      Host.symlink(src_path, destination_path)
-      destination_path
-    end
-
-    # @param installed_library_path [String] The library to query
-    # @return [Array<String>] Example sketch files
-    def library_examples(installed_library_path)
-      example_path = Pathname.new(installed_library_path) + "examples"
-      return [] unless File.exist?(example_path)
-
-      examples = example_path.children.select(&:directory?).map(&:to_path).map(&File.method(:basename))
-      files = examples.map do |e|
-        proj_file = example_path + e + "#{e}.ino"
-        proj_file.exist? ? proj_file.to_s : nil
-      end
-      files.reject(&:nil?).sort_by(&:to_s)
-    end
-  end
-end