samagotchi 0.149.0 → 0.151.0

data/lib/samagotchi/memory_bundle/remerge.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+require_relative "index_updater"
+require_relative "manifest"
+require_relative "memory_files"
+require_relative "merger"
+require_relative "provenance"
+require_relative "trash"
+require_relative "../model_overlay"
+
+module Samagotchi
+  module MemoryBundle
+    # The retry at the same version, memory files only: nothing else would come
+    # back to a file marked conflict until the bundle's next version. It merges
+    # what merges and settles those entries, leaving version, installed_at and
+    # the assets alone. Not the Installer: that rm_rfs guardrails, plugin and
+    # scripts, and GuardrailWiring#rules stats per tool call, so a live session
+    # could hit a transient deny.
+    #
+    # Each marked file, by what Merger::Upgrade says of it:
+    #   - not shipped now: its mark, base and text stay, and it reads as behind
+    #     unless level or missing, as every report has it;
+    #   - the shipped text isn't the one the mark names (a checkout at the same
+    #     VERSION can carry other text): left alone, behind as its mark says. A
+    #     mark that names none (older chi) is tried too, the migration; a real
+    #     conflict then records the digest and legacy_base, so
+    #     Merger.merge_outcome keeps protecting its file (a merge there may not
+    #     take a line out). A mark whose digest matches is tried on every run,
+    #     since the user's file or git may have changed;
+    #   - merged: replaced as the Installer does, base the incoming file;
+    #   - noop, keep (a merge that adds nothing too): the file has nothing left
+    #     to miss, so the mark goes;
+    #   - conflict: behind, and the mark records why, only when that changed;
+    #   - fast_forward (one Merger::Upgrade#outcome lets go ahead): the next
+    #     upgrade's (save_marked_file), behind until then;
+    #   - install: a missing file, which isn't behind (FileStanding.behind?): the
+    #     next upgrade or a restore puts it back, and a warning says so;
+    #   - one it can't read or replace: left as it is, behind, saying why.
+    #
+    # It can't loop: settling clears the mark, and a real conflict writes nothing.
+    # git runs once per marked file per run, and only for a marked file. A run
+    # is every chi start for the system bundle (SystemBundle.ensure!) and `chi
+    # update` for it and the other bundles chi ships; a bundle chi doesn't ship
+    # gets no Remerge, and `chi bundle upgrade` is its retry.
+    class Remerge
+      # merged: replaced (or, in a dry run, would be). settled: their mark gone
+      # with no merge. stamped: their mark was updated. behind: each file that
+      # stays behind, and why (FileStanding::Behind). work? says whether a real run
+      # would change anything.
+      Result = Data.define(:merged, :settled, :stamped, :behind, :warnings, :trash_copy) do
+        def work? = [merged, settled, stamped].any?(&:any?)
+      end
+      NOTHING = Result.new(merged: [], settled: [], stamped: [], behind: [], warnings: [], trash_copy: nil).freeze
+
+      # @param record [InstalledBundle, nil] the bundle's record (read when not given)
+      def initialize(name:, scope:, target_dir:, source_dir:, record: nil)
+        @name = name
+        @scope = scope
+        @target_dir = target_dir
+        @source_dir = source_dir
+        @provenance = Provenance.new(name: name)
+        @record = record || @provenance.record
+      end
+
+      # @param dry_run [Boolean] answer, writing nothing
+      # @return [Result]
+      def run(dry_run: false)
+        marked = @record ? @record.files.select { |_, entry| entry[:conflict] } : {}
+        return NOTHING if marked.empty?
+
+        @dry_run = dry_run
+        @merged = []
+        @settled = []
+        @stamped = []
+        @behind = []
+        @warnings = []
+        @trash = Trash.new(@name)
+        shipped = MemoryFiles.select(@source_dir, Manifest.read(dir: @source_dir))
+        marked.each { |key, entry| retry_file(key, entry, shipped) }
+        settle unless dry_run
+        Result.new(merged: @merged, settled: @settled, stamped: @stamped.map(&:file), behind: @behind, warnings: @warnings, trash_copy: @trash.copy_line)
+      end
+
+      private
+
+      def retry_file(key, entry, shipped)
+        behind = @record.behind_file?(key, view.paths(key)[:current])
+        unless shipped.include?(key)
+          not_shipped(key, entry) if behind
+          return
+        end
+
+        incoming_sha = Merger.digest(view.paths(key)[:incoming])
+        if entry[:incoming_sha] && entry[:incoming_sha] != incoming_sha
+          @behind << @record.behind_entry(key, entry) if behind
+          return
+        end
+
+        outcome = view.outcome(key)
+        case outcome.status
+        when :merged then merge_file(key, entry, outcome)
+        when :conflict then mark(key, entry, view.behind(key, outcome, @record.version))
+        when :noop, :keep then @settled << key
+        when :fast_forward then mark(key, entry, behind_for(key, FileStanding::WAITS_FOR_UPGRADE))
+        when :install then @warnings << "#{key}: #{FileStanding::FILE_MISSING}"
+        end
+      rescue SystemCallError => e
+        stay(key, FileStanding.left_as_it_is(e.message))
+      end
+
+      def behind_for(key, why) = FileStanding::Behind.new(file: key, version: @record.version, why: why)
+
+      # The record says it too, so chi bundle status reads as the row does. There
+      # is no bundle text to save or compare with: only the words can change.
+      def not_shipped(key, entry)
+        behind = behind_for(key, FileStanding::NOT_SHIPPED)
+        @behind << behind
+        @stamped << behind unless @record.behind_entry(key, entry) == behind
+      end
+
+      # Behind for now, with a reason worth no write to the record.
+      def stay(key, why) = @behind << behind_for(key, why)
+
+      # Behind, and the record says so: a write only when it has something new,
+      # so a real conflict at each chi start writes nothing.
+      def mark(key, entry, behind)
+        @behind << behind
+        @stamped << behind unless entry[:incoming_sha] && @record.behind_entry(key, entry) == behind && File.exist?(@provenance.incoming_path(key))
+      end
+
+      def merge_file(key, entry, outcome)
+        return @merged << key if @dry_run
+        return @merged << key if Merger.write_merged(view.paths(key)[:current], outcome, @trash)
+
+        mark(key, entry, behind_for(key, FileStanding.no_merge(TextMerge::RACED)))
+      end
+
+      def settle
+        files = @merged + @settled
+        @provenance.resolve_conflicts(files.to_h { |key| [key, view.paths(key)] }) unless files.empty?
+        @provenance.refresh_marks(@stamped, @source_dir) unless @stamped.empty?
+        @merged.each { |key| refresh_index(key) }
+        note_merges
+      end
+
+      def note_merges
+        @provenance.note_merged(@merged, version: @record.version, trash: @trash.dir) unless @merged.empty?
+      rescue StandardError => e
+        @warnings << "merge not noted on the record (#{e.message})"
+      end
+
+      # The file's size moved; its description and source stay. Not fatal, as
+      # in the Installer: the file is in place. An overlay has no line.
+      def refresh_index(key)
+        return if ModelOverlay.bundle_overlay?(key, bundle_files: @record.files.keys, target_dir: @target_dir)
+
+        IndexUpdater.update_index(@scope, key.delete_suffix(".md"), File.size(view.paths(key)[:current]), nil, source: @name)
+      rescue StandardError => e
+        @warnings << "index.md: line for #{key.delete_suffix(".md")} not updated (#{e.message})"
+      end
+
+      def view
+        @view ||= Merger::Upgrade.new(provenance: @provenance, target_dir: @target_dir, source_dir: @source_dir)
+      end
+    end
+  end
+end

data/lib/samagotchi/memory_bundle/shipped_history.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "yaml"
+require_relative "../chi_home"
+
+module Samagotchi
+  module MemoryBundle
+    # Whether a file's bytes are what some installed samagotchi gem shipped for
+    # that file of that bundle. A mark from an older chi may have the user's own
+    # text as its base (Merger.merge_outcome); bytes a chi shipped can't be that.
+    #
+    # Asked only when the guard would otherwise fire, so an ordinary start pays
+    # nothing. Reads the gem dirs by hand: the running process's (Gem.path, which
+    # a chi home's boot string sets) and the chi home's (a checkout or a stray
+    # GEM_PATH runs on other gems than the home's), never loading a gem or asking
+    # Gem::Specification. Anything it can't read is no match, and no match keeps
+    # the guard: the fallback is the safe side.
+    module ShippedHistory
+      GEM_DIR = /\Asamagotchi-#{Gem::Version::VERSION_PATTERN}(?:-[a-z0-9_-]+)?\z/
+      BUNDLES = File.join("lib", "samagotchi", "bundles")
+
+      module_function
+
+      # @param bundle_name [String] the name in the manifest: a shipped dir may
+      #   have another name in another version
+      # @param file_key [String] the file's name in the bundle
+      # @param path [String] the file holding the bytes asked about
+      # @return [Boolean]
+      def shipped?(bundle_name, file_key, path)
+        return false if file_key.include?("/")
+
+        bytes = File.binread(path)
+        gem_roots.any? { |root| holds?(root, bundle_name, file_key, bytes) }
+      rescue SystemCallError
+        false
+      end
+
+      # Each installed samagotchi gem's folder, wherever it is.
+      def gem_roots
+        gem_dirs.flat_map do |dir|
+          gems = File.join(dir, "gems")
+          Dir.children(gems).select { |name| name.match?(GEM_DIR) }.map { |name| File.join(gems, name) }
+        rescue SystemCallError
+          []
+        end
+      end
+
+      def gem_dirs = (Gem.path + home_gem_dirs).uniq
+
+      def home_gem_dirs
+        ChiHome.read&.gem_dirs || []
+      rescue StandardError
+        []
+      end
+
+      # The size first, the bytes next and the manifest last: 55 gems of 22
+      # bundles would be a thousand YAML files, and few candidates get that far.
+      def holds?(root, bundle_name, file_key, bytes)
+        bundles = File.join(root, BUNDLES)
+        Dir.glob("*", base: bundles).any? do |dir|
+          file = File.join(bundles, dir, file_key)
+          File.file?(file) && File.size(file) == bytes.bytesize && File.binread(file) == bytes &&
+            manifest_name(File.dirname(file)) == bundle_name
+        rescue SystemCallError
+          false
+        end
+      end
+
+      # The name a bundle's manifest gives, nil when it doesn't read. Not
+      # Manifest.read: an older manifest needn't pass this chi's checks.
+      def manifest_name(dir)
+        data = YAML.safe_load_file(File.join(dir, "manifest.yml"))
+        data.is_a?(Hash) ? data["name"] : nil
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/samagotchi/memory_bundle/shipped_update.rb

@@ -6,6 +6,7 @@ require_relative "manifest"
 require_relative "merger"
 require_relative "profile"
 require_relative "provenance"
+require_relative "remerge"
 require_relative "source"
 require_relative "status"
 require_relative "system_bundle"
@@ -22,8 +23,8 @@ module Samagotchi
     # newer and this chi meets its requires_chi. It never installs a
     # bundle that isn't installed and never downgrades.
     #
-    # .md files take the Installer's 3-way merge: a conflict keeps the
-    # user's file (kept). Hooks, rules and the plugin are replaced; an
+    # .md files take the Installer's 3-way merge: a clean text merge replaces
+    # the file (merged), a real conflict keeps the user's (behind). Hooks, rules and the plugin are replaced; an
     # edited one (which didn't load: its sha no longer matched) is listed
     # in replaced.
     #
@@ -32,15 +33,28 @@ module Samagotchi
     # to install (Profile.to_install, less the ones this chi can't load),
     # even at the same version, so a member that failed is retried. adds
     # names them.
+    #
+    # At the same version a row retries the files marked conflict (Remerge):
+    # :would_merge when that would change anything, then :merged when a file
+    # merged; what stays behind reads as behind whichever it is. Not retried for
+    # a project-scope bundle: chi bundle upgrade in the project.
     module ShippedUpdate
-      # status: :would_update (plan), :updated, :up_to_date, :skipped,
-      # :failed. note says why for skipped/failed. kept and replaced are
-      # file names; under plan they are what apply would keep or replace.
+      # status: :would_update, :would_merge (plan), :updated, :merged,
+      # :up_to_date, :skipped, :failed. note says why for skipped/failed, and
+      # what a retry at the same version warned of. merged and replaced are file
+      # names, behind FileStanding::Behind; under plan they are what apply would
+      # merge, replace or leave behind.
       # adds is nil but for a profile: the members it installs.
       # kept_own is nil but for a profile that installed a member: its
       # Profile.kept_own_line lines.
-      Row = Struct.new(:name, :from, :to, :status, :note, :kept, :replaced, :source_dir, :scope, :adds, :kept_own,
-                       keyword_init: true)
+      # trash_copy is nil but under apply when it saved edited files
+      # (Installer#trash_copy_line).
+      # earlier: the merge notes on its record when the plan read it
+      # (FileStanding::MergeNote): this run's own merges aren't among them.
+      Row = Struct.new(:name, :from, :to, :status, :note, :merged, :behind, :replaced, :source_dir, :scope, :adds, :kept_own,
+                       :trash_copy, :earlier, keyword_init: true) do
+        def kept = Array(behind).map(&:file)
+      end
 
       module_function
 
@@ -49,7 +63,11 @@ module Samagotchi
       def plan(shipped_dir: SourceNormalizer::SHIPPED_DIR, chi_version: Samagotchi::VERSION)
         shipped = Listing.shipped(dir: shipped_dir).to_h { |s| [s.name, s] }
         installed.map do |name, bundle|
-          plan_row(name, bundle, shipped[name], shipped_dir, chi_version)
+          row = plan_row(name, bundle, shipped[name], shipped_dir, chi_version)
+          # A row that retried nothing still has its files that stay behind.
+          row.behind = bundle.behind_files if row.status == :skipped && !bundle.error?
+          row.earlier = bundle.merge_notes
+          row
         end
       end
 
@@ -69,11 +87,13 @@ module Samagotchi
         installed.filter_map { |name, bundle| bundle unless bundle.error? || bundle.profile? || from_chi?(bundle, shipped[name]) }
       end
 
-      # Runs the Installer for each :would_update row; the others pass
-      # through. A failing bundle doesn't stop the rest.
+      # Runs the Installer for each :would_update row, Remerge for each
+      # :would_merge row; the others pass through. A failing bundle doesn't
+      # stop the rest.
       # @return [Array<Row>]
       def apply(rows)
         rows.map do |row|
+          next apply_remerge(row) if row.status == :would_merge
           next row unless row.status == :would_update
           next apply_profile(row) if row.adds
 
@@ -82,16 +102,32 @@ module Samagotchi
           installer.run
           row.dup.tap do |r|
             r.status = :updated
-            r.kept = installer.conflicts.keys
+            r.merged = installer.merged_files
+            r.behind = installer.behind
+            r.trash_copy = installer.trash_copy_line
           end
         rescue StandardError => e
           row.dup.tap do |r|
             r.status = :failed
             r.note = e.message.lines.first.to_s.strip
+            r.merged = installer&.merged_files || []
+            r.behind = installer&.behind || []
+            r.trash_copy = installer&.trash_copy_line
           end
         end
       end
 
+      def apply_remerge(row)
+        done = remerge(row).run
+        row.dup.tap do |r|
+          r.status = done.merged.empty? ? :up_to_date : :merged
+          r.merged = done.merged
+          r.behind = done.behind
+          r.trash_copy = done.trash_copy
+          r.note = note_of(done)
+        end
+      end
+
       def apply_profile(row)
         result = Profile.install(row.source_dir, shipped_dir: File.dirname(row.source_dir))
         row.dup.tap do |r|
@@ -111,7 +147,7 @@ module Samagotchi
       def from_chi?(bundle, ship) = !ship.nil? && bundle.source.to_s.match?(SourceNormalizer::SHIPPED_SOURCE)
 
       def plan_row(name, bundle, ship, shipped_dir, chi_version)
-        row = Row.new(name: name, kept: [], replaced: [])
+        row = Row.new(name: name, merged: [], behind: [], replaced: [])
         return skip(row, "manifest.json unreadable") if bundle.error?
 
         row.from = bundle.version.to_s
@@ -123,7 +159,7 @@ module Samagotchi
 
         row.source_dir = File.join(shipped_dir, ship.source)
         return profile_row(row, bundle, shipped_dir, chi_version) if Profile.shipped_meta?(row.source_dir, shipped_dir: shipped_dir)
-        return row.tap { |r| r.status = :up_to_date; r.to = nil } unless Listing.newer?(row.to, row.from)
+        return same_version_row(row, bundle) unless Listing.newer?(row.to, row.from)
 
         manifest = Manifest.read(dir: row.source_dir)
         if Manifest.requires_chi_failure(manifest.requires_chi, chi_version)
@@ -132,12 +168,43 @@ module Samagotchi
         return skip(row, "project scope: chi bundle upgrade #{ship.source} in the project") if row.scope == "project"
         return skip(row, "unknown scope #{row.scope}: upgrade chi or reinstall") unless MemoryPaths.scope_dir(row.scope)
 
-        row.kept = conflicts(row, bundle)
+        forecast = forecast(row, bundle)
+        row.merged = forecast.merged
+        row.behind = forecast.behind
         row.replaced = edited_executables(name, bundle)
         row.status = :would_update
         row
+      rescue SystemCallError => e
+        # A file it can't read: the upgrade would fail on it, so this is that
+        # row, not a crash of the whole plan. Apply passes it through.
+        row.status = :failed
+        row.note = Trash.failure_message(e)
+        row
+      end
+
+      # Up to date, unless a file marked conflict could settle now or stays behind.
+      def same_version_row(row, bundle)
+        row.to = nil
+        row.status = :up_to_date
+        if row.scope == "project" || !MemoryPaths.scope_dir(row.scope)
+          row.behind = bundle.behind_files
+          return row
+        end
+
+        planned = remerge(row).run(dry_run: true)
+        row.merged = planned.merged
+        row.behind = planned.behind
+        row.note = note_of(planned)
+        row.status = :would_merge if planned.work?
+        row
       end
 
+      def remerge(row)
+        Remerge.new(name: row.name, scope: row.scope, target_dir: MemoryPaths.scope_dir!(row.scope), source_dir: row.source_dir)
+      end
+
+      def note_of(remerged) = remerged.warnings.empty? ? nil : remerged.warnings.join("; ")
+
       def profile_row(row, bundle, shipped_dir, chi_version)
         manifest = Manifest.read(dir: row.source_dir)
         newer = Listing.newer?(row.to, row.from)
@@ -162,16 +229,14 @@ module Samagotchi
         row
       end
 
-      # The bundle's .md files whose local edits conflict with the shipped
-      # version (a same-name file it didn't install is skipped, not merged).
-      def conflicts(row, bundle)
-        provenance = Provenance.new(name: row.name)
-        target_dir = MemoryPaths.scope_dir!(row.scope)
-        owned = bundle.files.keys
-        (Dir.glob(File.join(row.source_dir, "*.md")).map { |f| File.basename(f) }.sort & owned).select do |key|
-          Merger.classify(base_path: provenance.base_path(key), current_path: File.join(target_dir, key),
-                          incoming_path: File.join(row.source_dir, key)) == :conflict
-        end
+      # What the upgrade would do to the bundle's .md files that local edits and
+      # the shipped version both changed (a same-name file it didn't install is
+      # skipped, not merged). Every shipped *.md, not MemoryFiles.select: see
+      # Merger::Upgrade#forecast.
+      def forecast(row, bundle)
+        shipped = Dir.glob(File.join(row.source_dir, "*.md")).map { |f| File.basename(f) }.sort
+        Merger::Upgrade.new(provenance: Provenance.new(name: row.name), target_dir: MemoryPaths.scope_dir!(row.scope),
+                            source_dir: row.source_dir).forecast(shipped & bundle.files.keys, version: row.to)
       end
 
       # Installed hooks, rules and the plugin whose sha differs from the

data/lib/samagotchi/memory_bundle/source.rb

@@ -2,7 +2,7 @@
 
 require "tmpdir"
 require "fileutils"
-require "shellwords"
+require_relative "../git_env"
 
 module Samagotchi
   module MemoryBundle
@@ -81,13 +81,13 @@ module Samagotchi
           args = ["git", "clone", "--depth", "1"]
           args += ["--branch", ref] if ref && !ref.empty?
           args += [url, clone_dir]
-          result = system(*args)
+          result = system(GitEnv::UNSET, *args)
           raise UnknownSourceError, "git clone failed for #{git_url}" unless result && File.directory?(clone_dir)
 
           # Capture HEAD commit before stripping .git (audit trail)
           commit = nil
           begin
-            commit = `git -C #{clone_dir.shellescape} rev-parse HEAD 2>/dev/null`.strip
+            commit = IO.popen(GitEnv::UNSET, ["git", "-C", clone_dir, "rev-parse", "HEAD"], err: File::NULL, &:read).strip
             commit = nil if commit.empty?
           rescue StandardError
             commit = nil

data/lib/samagotchi/memory_bundle/status.rb

@@ -18,7 +18,9 @@ module Samagotchi
       # disk, its installed plugin, why none of its hooks load, and its needs
       # with what this process's PATH found. The +files+ entries and +plugin+
       # stay Hashes: their fields differ per case (an unchecked file has no
-      # paths at all).
+      # paths at all). An entry's behind is its FileStanding::Behind, nil for a
+      # file that isn't; merge_note its FileStanding::MergeNote, nil without one;
+      # unreadable: the file is there, but this chi can't read it.
       Report = Data.define(:provenance, :scope, :scope_error, :target_dir, :files, :plugin,
                            :hooks_requires_failure, :needs)
 
@@ -34,14 +36,19 @@ module Samagotchi
         target_dir = MemoryPaths.scope_dir(scope)
         files = bundle.files
         details = {}
+        behind = bundle.behind_files(target_dir).to_h { |file| [file.file, file] }
+        notes = bundle.merge_notes.to_h { |note| [note.file, note] }
         files.each do |file_key_str, meta|
           next details[file_key_str] = unchecked_file(meta) unless target_dir
 
           target_path = File.join(target_dir, file_key_str)
           base_path = provenance.base_path(file_key_str)
           stored_checksum = meta[:checksum]
-          current_checksum = File.exist?(target_path) ? Digest::SHA256.hexdigest(File.read(target_path)) : nil
-          base_checksum = File.exist?(base_path) ? Digest::SHA256.hexdigest(File.read(base_path)) : nil
+          current_checksum = checksum(target_path)
+          unreadable = current_checksum == :unreadable
+          current_checksum = nil if unreadable
+          base_checksum = checksum(base_path)
+          base_checksum = nil if base_checksum == :unreadable
           modified = base_checksum && current_checksum && base_checksum != current_checksum
           missing = !File.exist?(target_path)
           # A model overlay has no index line (Installer#note_overlay).
@@ -49,6 +56,9 @@ module Samagotchi
           index_present = overlay ? nil : index_has_entry?(scope, file_key_str)
           details[file_key_str] = {
             conflict: meta[:conflict] == true,
+            behind: behind[file_key_str],
+            merge_note: notes[file_key_str],
+            unreadable: unreadable,
             stored_checksum: stored_checksum,
             current_checksum: current_checksum,
             base_checksum: base_checksum,
@@ -65,6 +75,14 @@ module Samagotchi
                    hooks_requires_failure: hooks_requires_failure(bundle), needs: needs_status(bundle))
       end
 
+      # The file's digest, nil when there is none, :unreadable when it can't be read.
+      def self.checksum(path)
+        File.exist?(path) ? Digest::SHA256.file(path).hexdigest : nil
+      rescue SystemCallError
+        :unreadable
+      end
+      private_class_method :checksum
+
       # Why none of the bundle's hooks load (this chi doesn't meet its
       # requires_chi, Hooks::BundleLoader), or nil: it has no hooks or meets it.
       def self.hooks_requires_failure(bundle)