MuranoCLI 3.2.0.beta.1 → 3.2.0.beta.5

data/lib/MrMurano/SyncUpDown-Core.rb

@@ -10,9 +10,12 @@ require 'pathname'
 require 'tempfile'
 require 'MrMurano/verbosing'
 require 'MrMurano/hash'
+require 'MrMurano/SyncUpDown-Item'
 
 module MrMurano
   module SyncCore
+    include Verbose
+
     #######################################################################
     # Methods that provide the core status/syncup/syncdown
 
@@ -20,7 +23,7 @@ module MrMurano
       if $cfg['tool.no-progress']
         say(msg)
       else
-        verbose(msg + "\n")
+        MrMurano::Verbose.verbose(msg + "\n")
       end
     end
 
@@ -158,8 +161,21 @@ module MrMurano
     # @param asdown [Boolean] Direction/prespective of diff
     # @return [String] The diff output
     def dodiff(merged, local, _there=nil, options={})
-      trmt = Tempfile.new([tolocalname(merged, @itemkey) + '_remote_', '.lua'])
-      tlcl = Tempfile.new([tolocalname(merged, @itemkey) + '_local_', '.lua'])
+      localname = tolocalname(merged, @itemkey)
+      # MUR-6488: Beware nested files, e.g., /tmp/js/script.js will
+      # not work, because /tmp/js does not exist!
+      # First, remove the leading path delimiter.
+      localname = localname.gsub(/^#{::File::SEPARATOR}*/, '')
+      # Next, replace remaining delimiters with a dot.
+      localname = localname.tr(::File::SEPARATOR, '.')
+      begin
+        # (lb): Not always Lua, e.g., might be PNG. Postfix doesn't matter, though.
+        trmt = Tempfile.new([localname + '_remote_', '.lua'])
+        tlcl = Tempfile.new([localname + '_local_', '.lua'])
+      rescue StandardError => err
+        MrMurano::Verbose.error("Failed to create temporary file: #{err}")
+        raise
+      end
       Pathname.new(tlcl.path).open('wb') do |io|
         # Copy the local file to a temp file, for the diff command.
         # And for resources, remove the local-only :selected key, as
@@ -234,29 +250,35 @@ module MrMurano
     # Check if an item matches a pattern.
     # @param items [Array<Item>] Of items to filter
     # @param patterns [Array<String>] Filters for _matcher
-    def _matcher(items, patterns)
+    def _matcher(items, patterns, desc)
       items.map do |item|
-        if patterns.empty?
-          item[:selected] = true
-        else
-          item[:selected] = patterns.any? do |pattern|
-            if pattern.to_s[0] == '#'
-              match(item, pattern)
-            else
-              lpath = _matcher_local_path(item)
-              _matcher_check_pattern(lpath, pattern)
-            end
-          end
-        end
+        item[:selected] = _matcher_check_patterns(item, patterns)
         item
       end
+      _matcher_debug_log(items, desc)
+      items
     end
     private :_matcher
 
-    def _matcher_local_path(item)
+    def _matcher_check_patterns(item, patterns)
+      return true if patterns.empty?
+      patterns.any? do |pattern|
+        if pattern.to_s[0] == '#'
+          match(item, pattern)
+        else
+          lpath = _matcher_local_path(item)
+          _matcher_check_pattern(lpath, pattern)
+        end
+      end
+    end
+    private :_matcher_check_patterns
+
+    def _matcher_local_path(item, friendly: false)
       if !defined?(item.local_path) || item.local_path.nil?
         into = location
         tolocalpath(into, item)
+      elsif friendly
+        item.location_friendly(full_path: true)
       else
         item[:local_path]
       end
@@ -275,14 +297,27 @@ module MrMurano
       # Be nice and try globbing the start of the path. E.g., if user
       # specifies pattern=relative/path/to/file.rb, we should retry their
       # request as */relative/path/to/file.rb. But don't glob if it looks
-      # like the user is already globbing or using absolutes.
+      # like the user is using an /absolute/path.
       return false if pattern.to_s.start_with?(File::SEPARATOR)
-      # See if the pattern includes any '*' that are not \-delimited.
-      return false if pattern.to_s =~ /(?<!\\)\*/
       lpath.fnmatch? "*/#{pattern}"
     end
     private :_matcher_check_pattern
 
+    def _matcher_debug_log(items, desc)
+      # If :dup_count is set, there are duplicates of the same item.
+      # Note that the item with dup_count == 0 is a control item, which
+      # we ignore (it was cloned from another item also in the list).
+      culled = items.reject do |item|
+        !item[:dup_count].nil? && item[:dup_count].zero?
+      end
+      items_loci = culled.map do |item|
+        _matcher_local_path(item, friendly: true).to_s
+      end
+      items_list = items_loci.sort.join("\n  ")
+      debug "#{self.class}: #{desc}'s matches:\n  #{items_list}"
+    end
+    private :_matcher_debug_log
+
     ## Get status of things here verses there
     #
     # @param options [Hash, Commander::Command::Options] Options on operation
@@ -306,7 +341,17 @@ module MrMurano
 
       items_cull_clashes!(statuses)
 
-      statuses.each_value { |items| select_selected(items) } unless options[:unselected]
+      # 2018-04-24: (lb): How did I not see this til now? :unselected is :notimplemented.
+      unless options[:unselected]
+        statuses.each do |bucket, items|
+          select_selected(items)
+          next unless $cfg['tool.debug']
+          loci = items.map { |item| item.location_friendly(full_path: true) }
+          selected_paths = loci.sort.join("\n  ")
+          selected_paths = (selected_paths.empty? && ' <none>' || "\n  ") + selected_paths
+          debug %(#{self.class}: Selected #{bucket} items:#{selected_paths})
+        end
+      end
 
       statuses
     end
@@ -315,8 +360,8 @@ module MrMurano
       # Get the solution name from the config.
       # Convert, e.g., application.id => application.name
       soln_name = $cfg[@solntype.gsub(/(.*)\.id/, '\1.name')]
-      # Skip this syncable if the api_id is not set, or if user wants to skip
-      # by solution.
+      # Skip this syncable if the api_id is not set,
+      # or if the user wants to skip by solution.
       skip_sol = false
       if !api_id? ||
          (options[:type] == :application && @solntype != 'application.id') ||
@@ -364,7 +409,7 @@ module MrMurano
       end
       return nil unless skip_sol
       ret = { toadd: [], todel: [], tomod: [], unchg: [], skipd: [], clash: [] }
-      ret[:skipd] << { synckey: self.class.description }
+      ret[:skipd] << SyncUpDown::Item.new(synckey: self.class.description)
       ret
     end
 
@@ -384,31 +429,54 @@ module MrMurano
 
       resolve_config_var_usage!(there, local)
 
-      there = _matcher(there, selected)
-      local = _matcher(local, selected)
+      debug "#{self.class}: user-specified patterns:\n  #{selected.sort.join("\n  ")}"
+      there = _matcher(there, selected, :there)
+      local = _matcher(local, selected, :local)
 
-      therebox = {}
-      there.each do |item|
-        item[:synckey] = synckey(item)
-        item[:synctype] = self.class.description
-        therebox[item[:synckey]] = item
-      end
+      therebox, theredup = items_classify_and_find_duplicates(there)
+      localbox, localdup = items_classify_and_find_duplicates(local)
+      items_log_duplicates_there_local(theredup, localdup)
+
+      localbox = resurrect_undeletables(localbox, therebox)
 
-      localbox = {}
-      local.each do |item|
+      [therebox, localbox]
+    end
+
+    def items_classify_and_find_duplicates(items)
+      lkup = {}
+      dupl = {}
+      items.each do |item|
         skey = synckey(item)
-        # 2017-07-02: Check for local duplicates.
         unless item[:dup_count].nil? || item[:dup_count].zero?
+          dupl[skey] = [] if dupl[skey].nil?
+          dupl[skey].push item
           skey += "-#{item[:dup_count]}"
         end
         item[:synckey] = skey
         item[:synctype] = self.class.description
-        localbox[skey] = item
+        lkup[skey] = item
       end
+      [lkup, dupl]
+    end
 
-      localbox = resurrect_undeletables(localbox, therebox)
+    def items_log_duplicates_there_local(theredup, localdup)
+      if !theredup.empty? && $cfg['tool.developer']
+        error %(Unexpected: Duplicate items detected in "there" items!)
+      end
+      items_log_duplicates(theredup)
+      items_log_duplicates(localdup)
+    end
 
-      [therebox, localbox]
+    def items_log_duplicates(dupl)
+      dupl.each do |skey, items|
+        msg = 'Duplicate definitions found for '
+        if self.class.description.to_s != ''
+          msg += "#{fancy_ticks(self.class.description)} item: "
+        end
+        msg += fancy_ticks(skey)
+        warning(msg)
+        items.each { |item| warning(" #{item.location_friendly}") }
+      end
     end
 
     def items_new_and_old!(statuses, options, therebox, localbox)
@@ -424,54 +492,94 @@ module MrMurano
     end
 
     def items_mods_and_chgs!(statuses, options, therebox, localbox)
-      tomod = []
-      unchg = []
-      toadd = []
-      todel = []
-
+      init_mods_and_chgs_arrs(statuses)
+      verbose "#{self.class}: Checking for changes:"
       (localbox.keys & therebox.keys).each do |key|
         local = localbox[key]
         there = therebox[key]
+        verbose %(  #{local.location_friendly(full_path: true)})
         # Skip this item if it's got duplicate conflicts.
         next if !local.is_a?(Hash) && local.dup_count == 0
-        if options[:asdown]
-          # Want 'there' to override 'local'.
-          mrg = local.merge(there)
-        else
-          # Want 'local' to override 'there' except for itemkey.
-          mrg = local.reject { |k, _v| k == @itemkey.to_sym }
-          mrg = there.merge(mrg)
-        end
+        mrg = item_local_there_merged(local, there, options)
+        # MAYBE/2018-04-24: (lb): If both are not :selected, do not bother diffing.
+        dirty_phantom = item_merged_set_status(mrg, statuses, local, there, options)
+        next unless dirty_phantom
+        localbox.delete(key)
+      end
+    end
 
-        if docmp(local, there)
-          if (options[:diff] || local[:phantom] || local[:undeletable]) && mrg[:selected]
-            mrg[:diff] = dodiff(mrg.to_h, local, there, options)
-            if mrg[:diff].to_s.empty?
-              debug %(Clean diff: #{local[:synckey]})
-              mrg[:diff] = '<Nothing changed (was timestamp difference)>'
-              unchg << mrg
-            elsif local[:phantom]
-              if options[:asdown]
-                toadd << mrg
-              else
-                todel << mrg
-              end
-              localbox.delete(key)
-            else
-              tomod << mrg
-            end
-          else
-            tomod << mrg
-          end
+    def init_mods_and_chgs_arrs(statuses)
+      statuses[:tomod] = []
+      statuses[:unchg] = []
+      # SKIP: statuses[:toadd]
+      # SKIP: statuses[:todel]
+    end
+
+    def item_local_there_merged(local, there, options)
+      if options[:asdown]
+        # Want 'there' to override 'local'.
+        local.merge(there)
+      else
+        # Want 'local' to override 'there' except for itemkey.
+        mrg = local.reject { |k, _v| k == @itemkey.to_sym }
+        there.merge(mrg)
+      end
+    end
+
+    def item_merged_set_status(mrg, statuses, local, there, options)
+      if !docmp(local, there)
+        # Nothing dirty, according to cursory compare using timestamps, hashes, etc.
+        statuses[:unchg] << mrg
+        false
+      else
+        # Do an in-depth compare, e.g., content vs content, and return if phantom.
+        item_dirty_set_status(mrg, statuses, local, there, options)
+      end
+    end
+
+    def item_dirty_set_status(mrg, statuses, local, there, options)
+      if !(options[:diff] || local[:phantom] || local[:undeletable])
+        # NOTE: (lb): We mark an item `modified` if any of these are true:
+        #   1. User is not diffing, e.g., murano-status, or sync-up/-down commands.
+        #   2. The local item is a phantom item, i.e., user deleted locally, but
+        #      platform does not let us really delete (we just clear contents),
+        #      so we maintain a local `phantom` item to represent this.
+        #   3. The item is undeletable. (Checking this might make checking
+        #      if phantom redundant, since only undeletables can be phantom,
+        #      I think.)
+        statuses[:tomod] << mrg
+        # Not a phantom item; return false.
+        false
+      else
+        item_merged_diff_status(mrg, statuses, local, there, options)
+      end
+    end
+
+    def item_merged_diff_status(mrg, statuses, local, there, options)
+      dirty_phantom = false
+      mrg[:diff] = dodiff(mrg.to_h, local, there, options)
+      if mrg[:diff].to_s.empty?
+        # Nothing changed -- cursory compare was false positive.
+        # FIXME: Update cache or otherwise prevent from recurring.
+        debug %(    Clean diff: #{local[:synckey]})
+        mrg[:diff] = '<Nothing changed (was timestamp difference)>'
+        statuses[:unchg] << mrg
+      elsif local[:phantom]
+        # Only exists local or remote, not both, and dirty diff.
+        # I.e., one side may no longer be phantom (might have content).
+        dirty_phantom = true
+        debug %(    Drty Phntm: #{local[:synckey]})
+        if options[:asdown]
+          statuses[:toadd] << mrg
         else
-          unchg << mrg
+          statuses[:todel] << mrg
         end
+      else
+        # Dirty diff.
+        debug %(    Dirty diff: #{local[:synckey]})
+        statuses[:tomod] << mrg
       end
-
-      statuses[:tomod] = tomod
-      statuses[:unchg] = unchg
-      statuses[:toadd] += toadd
-      statuses[:todel] += todel
+      dirty_phantom
     end
 
     def sort_by_name(list)

data/lib/MrMurano/SyncUpDown-Item.rb

@@ -25,7 +25,7 @@ module MrMurano
       # @return [String] The Lua code for this item. (not all items use this.)
       attr_accessor :script
       # @return [Integer] The line in #local_path where this #script starts.
-      attr_accessor :line
+      attr_accessor :line_beg
       # @return [Integer] The line in #local_path where this #script ends.
       attr_accessor :line_end
       # @return [String] If requested, the diff output.
@@ -80,9 +80,14 @@ module MrMurano
       def []=(key, value)
         public_send("#{key}=", value)
       rescue StandardError => err
-        MrMurano::Verbose.error(
-          "Unable to set key: #{key} / value: #{value} / err: #{err} / self: #{inspect}"
-        )
+        # DEVS: (lb): This isn't an error so much as whatever Item class this
+        # is is missing some members (perhaps they were added recently to the
+        # service, and the CLI has yet to be updated).
+        if $cfg['tool.developer']
+          MrMurano::Verbose.error(
+            "Unable to set key: #{key} / value: #{value} / err: #{err} / self: #{inspect}"
+          )
+        end
       end
 
       # Delete a key
@@ -153,6 +158,26 @@ module MrMurano
         #   MAYBE/2017-07-18: Permanently disable Style/RedundantSelf?
         self.to_h <=> other.to_h
       end
+
+      def location_friendly(show_type: false, full_path: false)
+        if !@local_path.nil?
+          desc = @local_path
+          desc = desc.relative_path_from(Pathname.pwd) unless full_path
+          desc = desc.to_s
+          desc = "#{desc}:#{@line_beg}" if !@line_beg.nil? && @line_beg > 0
+          desc = "#{desc}-#{@line_end}" if !@line_end.nil? && @line_end > 0
+        else
+          desc = @synckey
+        end
+        desc += ' [phantom]' if @phantom
+        return desc unless show_type
+        unless @pp_desc.to_s.empty? || (@pp_desc == @synckey)
+          desc += " (#{@pp_desc})"
+        end
+        # NOTE: This path is the endpoint path.
+        desc += " [#{@method} #{@path}]" if (@method && @path)
+        desc
+      end
     end
   end
 end

data/lib/MrMurano/SyncUpDown.rb

@@ -22,6 +22,7 @@ module MrMurano
   module SyncUpDown
     include SyncAllowed
     include SyncCore
+    include Verbose
 
     #######################################################################
     # Methods that must be overridden
@@ -186,7 +187,9 @@ module MrMurano
       # than downloading again.
       local.dirname.mkpath
       local.open('wb') do |io|
-        fetch(id) do |chunk|
+        # Do not modify remote content when diffing, e.g., do not add #ENDPOINT header.
+        untainted = options[:diff]
+        fetch(id, untainted) do |chunk|
           io.write config_vars_encode chunk
         end
       end
@@ -211,7 +214,7 @@ module MrMurano
         # (lb): This is okay on Windows. (We really need a better solution.)
         unless OS.windows?
           msg = 'Unexpected: touch failed on non-Windows machine'
-          warn "#{msg} / host_os: #{RbConfig::CONFIG['host_os']} / err: #{err}"
+          warning "#{msg} / host_os: #{RbConfig::CONFIG['host_os']} / err: #{err}"
         end
       end
     end
@@ -329,19 +332,13 @@ module MrMurano
           items[skey] = item.clone
           items[skey][:dup_count] = 0
         end
-        counts[skey] = counts.key?(skey) && counts[skey] + 1 || 1
+        counts[skey] = counts.key?(skey) && (counts[skey] + 1) || 1
         # Use a unique synckey so all duplicates make it in the list.
         uniq_synckey = "#{skey}-#{counts[skey]}"
         item[:dup_count] = counts[skey]
         # This sets the alias for the output, so duplicates look unique.
         item[@itemkey.to_sym] = uniq_synckey
         items[uniq_synckey] = item
-        msg = "Duplicate definition found for #{fancy_ticks(skey)}"
-        if self.class.description.to_s != ''
-          msg += " for #{fancy_ticks(self.class.description)}"
-        end
-        warning(msg)
-        warning(" #{item.local_path}")
       else
         items[skey] = item
       end
@@ -412,7 +409,8 @@ module MrMurano
       debug "#{self.class}: Getting local items from:\n  #{from}"
       search_in = from.to_s
       sf = searchFor.map { |i| ::File.join(search_in, i) }
-      debug "#{self.class}: Globs:\n  #{sf.join("\n  ")}"
+      debug "#{self.class}: #{@project_section}.include:\n  #{sf.sort.join("\n  ")}"
+      debug "#{self.class}: #{@project_section}.exclude:\n  #{ignoring.sort.join("\n  ")}"
       # 2017-07-27: Add uniq to cull duplicate entries that globbing
       # all the ways might produce, otherwise status/sync/diff complain
       # about duplicate resources. I [lb] think this problem has existed
@@ -449,7 +447,9 @@ module MrMurano
       #items = items.flatten.compact.sort_by!(&:local_path)
       #debug "#{self.class}: items:\n  #{items.map(&:local_path).join("\n  ")}"
       items = items.flatten.compact.sort_by { |it| it[:local_path] }
-      debug "#{self.class}: items:\n  #{items.map { |it| it[:local_path] }.join("\n  ")}"
+      loci = items.map { |it| it.location_friendly(full_path: true) }
+      item_list = loci.sort.join("\n  ")
+      debug "#{self.class}: localitems' matches:\n  #{item_list}"
       sort_by_name(items)
     end
 

data/lib/MrMurano/Webservice-Cors.rb

@@ -36,7 +36,7 @@ module MrMurano
         #@project_section = :cors
       end
 
-      def fetch(_id=nil)
+      def fetch(_id=nil, _untainted=false)
         ret = get
         return [] unless ret.is_a?(Hash) && !ret.key?(:error)
         if ret.key?(:cors)

data/lib/MrMurano/Webservice-Endpoint.rb

@@ -60,7 +60,7 @@ module MrMurano
         #   sort_by_name(ret)
       end
 
-      def fetch(id)
+      def fetch(id, untainted=false)
         ret = get('/' + id.to_s)
         unless ret.is_a?(Hash) && !ret.key?(:error)
           error "#{UNEXPECTED_TYPE_OR_ERROR_MSG}: #{ret}"
@@ -69,18 +69,32 @@ module MrMurano
 
         ret[:content_type] = 'application/json' if ret[:content_type].empty?
 
-        add_terminating_nl = ret[:script].end_with? "\n"
-        script = ret[:script].lines.map(&:chomp)
+        if untainted
+          script = ret[:script]
+        else
+          add_terminating_nl = ret[:script].end_with? "\n"
+          script = ret[:script].lines.map(&:chomp)
+          fetch_fix_script_header(script, ret)
+          script = script.join("\n")
+          script += "\n" if add_terminating_nl
+        end
+
+        if block_given?
+          yield script
+        else
+          script
+        end
+      end
 
+      def fetch_fix_script_header(script, ret)
         aheader = (script.first || '')
 
         rh = ['--#ENDPOINT', ret[:method].upcase, ret[:path]]
         rh << ret[:content_type] if ret[:content_type] != 'application/json'
         rheader = rh.join(' ')
 
-        # if header is missing add it.
+        # If header is missing add it.
         # If header is wrong, replace it.
-
         md = @match_header.match(aheader)
         if md.nil?
           # header missing.
@@ -95,13 +109,7 @@ module MrMurano
         end
         # otherwise current header is good.
 
-        script = script.join("\n")
-        script += "\n" if add_terminating_nl
-        if block_given?
-          yield script
-        else
-          script
-        end
+        script
       end
 
       ##
@@ -167,12 +175,13 @@ module MrMurano
         items = []
         path = Pathname.new(path) unless path.is_a? Pathname
         cur = nil
-        lineno = 0
+        lineno = 1
         path.readlines.each do |line|
+          lineno += 1
           md = @match_header.match(line)
           if !md.nil?
             # header line.
-            cur[:line_end] = lineno unless cur.nil?
+            cur[:line_end] = lineno - 1 unless cur.nil?
             items << cur unless cur.nil?
             # VERIFY/2017-07-03: The syncdown test is revealing a
             #   problem with casing. The original file has a lowercase
@@ -193,14 +202,13 @@ module MrMurano
               path: md[:path],
               content_type: (md[:ctype] || 'application/json'),
               local_path: path,
-              line: lineno,
+              line_beg: lineno,
               script: up_line,
             )
           elsif !cur.nil? && !cur[:script].nil?
             # 2017-07-02: Frozen string literal: change << to +=
             cur[:script] += line
           end
-          lineno += 1
         end
         cur[:line_end] = lineno unless cur.nil?
         items << cur unless cur.nil?
@@ -234,7 +242,10 @@ module MrMurano
         if item_b[:script].nil? && item_b[:local_path]
           item_b[:script] = item_b[:local_path].read
         end
-        (item_a[:script] != item_b[:script] || item_a[:content_type] != item_b[:content_type])
+        cmp = false
+        cmp ||= item_a[:content_type] != item_b[:content_type]
+        cmp ||= item_a[:script] != item_b[:script]
+        cmp
       end
     end