saboteur 0.3.0 → 0.5.0

data/lib/saboteur/engine/candidates.rb

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require "prism"
+require_relative "../inline_suppression"
 
 module Saboteur
   class Engine
@@ -12,7 +13,7 @@ module Saboteur
       def self.build_sources(config)
         root = File.realpath(config.root)
         config["sources"].map do |entry|
-          next MutationSource::Builtin.new(arid: config["arid"]) if entry == "builtin"
+          next MutationSource::Builtin.new(arid: config["arid"], operators: config["operators"]) if entry == "builtin"
 
           file, const = entry.split(":", 2)
           raise ConfigError, "mutation source #{entry.inspect} must be builtin or path.rb:ClassName" unless const
@@ -22,37 +23,88 @@ module Saboteur
         end
       end
 
-      def initialize(sources, diagnostics)
+      def initialize(sources, diagnostics, hazard_guard: nil, bytecode: BytecodeEquivalence.new)
         @sources = sources
         @diagnostics = diagnostics
+        @hazard_guard = hazard_guard
+        @bytecode = bytecode
       end
 
       def for(subject)
-        return [] unless parses?(subject.source)
+        unless parses?(subject.source)
+          @diagnostics.record_generation(subject, source: "subject", reason: "invalid_subject_syntax")
+          return []
+        end
 
         seen = Set[subject.source]
-        list = @sources.flat_map { |source| source.mutations_for(subject) }.filter_map do |mutation|
+        list = @sources.flat_map { |source| proposals(source, subject) }.filter_map do |mutation|
           candidate(subject, mutation, seen)
         end
+        list = suppress_inline(list, subject)
+        list = suppress_hazards(list, subject)
         list.sort_by.with_index { |(mutation, _), index| [mutation.coarse ? 0 : 1, index] }
       end
 
       private
 
+      def suppress_inline(list, subject)
+        directives = InlineSuppression.new(subject.file, subject.file_source)
+        list.reject do |mutation, _text|
+          directive = directives.reason(subject, mutation)
+          @diagnostics.record_inline_suppression(subject, mutation, directive) if directive
+          directive
+        end
+      end
+
+      # The hazard guard (issue #102) refuses mutants that would leave a
+      # destructive call running with altered inputs; each refusal is counted.
+      def suppress_hazards(list, subject)
+        return list unless @hazard_guard
+
+        kept, suppressed = @hazard_guard.partition(list, subject)
+        suppressed.each { |(mutation, _)| @diagnostics.record_hazard_suppression(subject, mutation) }
+        kept
+      end
+
+      def proposals(source, subject)
+        proposals = source.mutations_for(subject)
+        if proposals.empty?
+          @diagnostics.record_generation(subject, source: source_name(source), reason: "empty_generation")
+        end
+        proposals
+      end
+
+      def source_name(source)
+        name = source.name if source.respond_to?(:name)
+        name || "anonymous"
+      end
+
+      def record_drop(subject, mutation, reason)
+        @diagnostics.record_generation(subject, source: mutation.source, operator: mutation.operator, reason: reason)
+        nil
+      end
+
       def candidate(subject, mutation, seen)
         text = subject.apply(mutation.edits)
-        return if seen.include?(text)
+        return record_drop(subject, mutation, "unchanged") if text == subject.source
+        return record_drop(subject, mutation, "duplicate") if seen.include?(text)
 
         seen << text
         unless parses?(text)
           @diagnostics.count_invalid_mutation
-          return
+          return record_drop(subject, mutation, "invalid_syntax")
         end
 
+        return record_drop(subject, mutation, "equivalent_bytecode") if equivalent?(subject, text)
+
         [mutation, text]
       rescue ArgumentError => e
         @diagnostics.warn("#{subject.id}: dropped #{mutation.source} mutation: #{e.message}")
-        nil
+        record_drop(subject, mutation, "invalid_edit")
+      end
+
+      def equivalent?(subject, text)
+        @bytecode.equivalent?(subject.source, text, frozen_string_literal: subject.frozen_literals)
       end
 
       def parses?(text)

data/lib/saboteur/engine/coverage_store.rb

@@ -15,22 +15,25 @@ module Saboteur
       LOCKFILES = %w[Gemfile.lock gems.locked].freeze
       LOG = "coverage-build.log"
 
-      def initialize(config:, integration:, hooks:, supervisor:, diagnostics:)
+      def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, load_lines: {})
         @config = config
         @integration = integration
         @hooks = hooks
         @supervisor = supervisor
         @diagnostics = diagnostics
+        @load_lines = load_lines
         @root = File.realpath(config.root)
       end
 
       def fetch(files)
         path = File.join(@config.path("cache_dir"), "coverage-#{cache_key(files)}.json")
         cached = @config["cache"] ? Selection::CoverageMap.load(path) : nil
-        return cached if cached
+        return refuse_empty_map(cached, files) if cached
 
-        map = build(files)
+        map = refuse_empty_map(build(files), files)
         map.save(path)
+        File.write(File.join(@config.path("cache_dir"), "preview.json"),
+                   JSON.generate(path: path, fingerprint: RunFingerprint.new(config: @config).snapshot.digest))
         map
       end
 
@@ -52,18 +55,62 @@ module Saboteur
                        "Run through the saboteur executable and disable SimpleCov for the run."
         end
 
-        @diagnostics.inform("building coverage map (runs the whole suite once)")
+        scope = @config["tests"].empty? ? "the whole suite" : "the selected specs"
+        @diagnostics.inform("building coverage map (runs #{scope} once)")
         # The suite child forked next inherits this heap; pack it first, to
         # reduce later copying of shared pages.
         Execution::ForkReadiness.prepare_heap
-        result = @supervisor.run(timeout: @config["coverage_timeout"]) { |writer| record(writer, files) }
+        heartbeat = @diagnostics.progress_heartbeat("building coverage map").watch(log_path)
+        result = @supervisor.run(timeout: @config["coverage_timeout"], on_tick: heartbeat.method(:tick)) do |writer|
+          record(writer, files)
+        end
+        map_from(result)
+      end
+
+      def map_from(result)
         raise Error, "coverage map build stopped: #{result.detail}" if result.stopped_by
         raise Error, "coverage map build failed: #{died(result.exit_status)}" if result.payload.empty?
 
         data = JSON.parse(result.payload)
         raise Error, "coverage map build failed: #{data['error']}" if data["error"]
 
-        Selection::CoverageMap.from_json(result.payload)
+        with_parent_load_lines(Selection::CoverageMap.from_json(result.payload))
+      end
+
+      # The parent's load lines merge into the freshly built map: the forked
+      # build child cannot inherit the parent's Coverage counters.
+      def with_parent_load_lines(map)
+        @load_lines.empty? ? map : map.with_load_lines(@load_lines)
+      end
+
+      # A whole-suite build that records nothing - no line hit and no method
+      # call - in any included file fails the run (exit 3): usually no test
+      # in the suite exercises these files, or the setup hid the app from
+      # Coverage (Bootsnap's compile cache is the default Rails case). An
+      # all-endless-def file records method calls only and passes. Either way
+      # a run on a truly empty map would test nothing, and exit 0 would lie.
+      # With explicit tests (--test) an empty map is a legitimate outcome
+      # instead: the user narrowed the suite to specs that may not touch
+      # these sources, so the map passes with a warning. Applies to a cached
+      # map too, so a map cached before this check cannot keep a run green,
+      # and the guard runs before saving, so an empty map is never cached.
+      def refuse_empty_map(map, files)
+        return map if files.empty? || !map.empty?
+
+        if @config["tests"].empty?
+          raise Error, "Coverage recorded nothing in #{files.size} included files during a whole-suite " \
+                       "build. No test in the suite may exercise these files, or the setup hid the app " \
+                       "from Coverage#{bootsnap_note}. Run `saboteur doctor`."
+        end
+        @diagnostics.warn("Coverage recorded nothing in #{files.size} included files; " \
+                          "the selected specs may not exercise them.")
+        map
+      end
+
+      # Bootsnap is named only when it is loaded: blaming a library the app
+      # does not use sends the reader to the wrong place.
+      def bootsnap_note
+        " (Bootsnap's compile cache is the default Rails case)" if defined?(::Bootsnap)
       end
 
       def died(status)

data/lib/saboteur/engine/data_inputs.rb

@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class Engine
+    # Non-Ruby files a test run can read and that change its outcome: Rails
+    # fixtures, schema dumps and database/environment configuration, plus
+    # anything listed under `fingerprint_extra`. Ruby files are tracked
+    # elsewhere; these have no .rb extension, so the Ruby walk misses them.
+    # An absent pattern simply matches nothing, so plain gems are unaffected.
+    module DataInputs
+      GLOBS = %w[
+        spec/fixtures/**/*
+        spec/**/fixtures/**/*
+        test/fixtures/**/*
+        db/structure.sql
+        db/schema.rb
+        config/database.yml
+        config/database.yml.erb
+        config/*.yml
+        config/environments/*.yml
+        config/credentials/*.yml.enc
+      ].freeze
+
+      # True when the absolute path is one of the declared inputs, whether or
+      # not the file exists yet. A run must never write there.
+      def self.match?(root, path, extra: [])
+        return false unless path.start_with?("#{root}/")
+
+        relative = path.delete_prefix("#{root}/")
+        flags = File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB
+        (GLOBS + extra).any? { |pattern| File.fnmatch?(pattern, relative, flags) }
+      end
+
+      # Sorted absolute paths of regular files (symlinks followed) under root.
+      def self.files(root, extra: [])
+        (GLOBS + extra).flat_map { |pattern| Dir.glob(pattern, File::FNM_DOTMATCH, base: root) }
+                       .map { |relative| File.join(root, relative) }
+                       .select { |path| File.file?(path) }.uniq.sort
+      end
+    end
+  end
+end

data/lib/saboteur/engine/diagnostics.rb

@@ -1,19 +1,90 @@
 # SPDX-License-Identifier: MIT
 # frozen_string_literal: true
 
+require "tempfile"
+
 module Saboteur
   class Engine
     # What a run has to say besides results: warnings for the report, the
     # count of mutations dropped for not parsing, and progress marks.
     class Diagnostics
+      # See progress_heartbeat. Time-gated itself, so the supervisor can
+      # call tick on every poll. A phase whose child writes "N/M examples
+      # finished" lines to a log (the baseline and coverage children do) can
+      # be watched: the heartbeat then adds the count, the percentage and an
+      # estimate of the time left to its elapsed-time line.
+      class ProgressHeartbeat
+        COUNT = %r{(\d+)/([1-9]\d*) examples finished}
+        TAIL_BYTES = 4096
+
+        def initialize(diagnostics, label, interval: 30)
+          @diagnostics = diagnostics
+          @label = label
+          @interval = interval
+          @started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          @last_print = @started
+        end
+
+        # Reads progress from the end of this log on each print. Returns self.
+        def watch(log_path)
+          @log = log_path
+          self
+        end
+
+        def tick
+          now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          return if now - @last_print < @interval
+
+          @last_print = now
+          elapsed = now - @started
+          @diagnostics.inform("#{@label}: #{elapsed.round} s elapsed#{progress_note(elapsed)}")
+        end
+
+        private
+
+        # ", 250/1000 examples (25 %), about 90 s left", or "" when the log has
+        # no count yet. The estimate assumes examples take equally long, so it
+        # is rough: slow examples late in the order make it too low.
+        def progress_note(elapsed)
+          done, total = examples_finished
+          return "" if done.nil?
+
+          note = ", #{done}/#{total} examples (#{(done * 100.0 / total).floor} %)"
+          return note if done.zero? || done >= total
+
+          "#{note}, about #{(elapsed * (total - done) / done).ceil} s left"
+        end
+
+        def examples_finished
+          return unless @log
+
+          tail = File.read(@log, TAIL_BYTES, [File.size(@log) - TAIL_BYTES, 0].max).to_s.scrub
+          tail.scan(COUNT).last&.map(&:to_i)
+        rescue SystemCallError
+          nil
+        end
+      end
+
       MARKS = { killed: ".", survived: "S", timeout: "T", resource_limit: "L", error: "E" }.freeze
+      LEGEND = ". killed  S survived  T timeout  L resource limit  E error"
+      LINE_WIDTH = 50
 
-      attr_reader :warnings, :invalid_mutations
+      GenerationEvent = Data.define(:file, :subject, :source, :operator, :reason)
+      Omission = Data.define(:subject, :count, :limit)
+      DiscoveryFailure = Data.define(:file, :reason)
+
+      attr_reader :warnings, :invalid_mutations, :generation_events, :omissions, :discovery_failures,
+                  :hazard_suppressions, :inline_suppressions
 
       def initialize(err)
         @err = err
         @warnings = []
         @invalid_mutations = 0
+        @generation_events = []
+        @omissions = []
+        @discovery_failures = []
+        @hazard_suppressions = 0
+        @inline_suppressions = []
       end
 
       def warn(message)
@@ -24,14 +95,117 @@ module Saboteur
         @invalid_mutations += 1
       end
 
+      def record_generation(subject, source:, reason:, operator: nil)
+        values = [subject.file, subject.id, source, operator, reason].map do |value|
+          value.nil? ? nil : value.to_s.dup.freeze
+        end
+        @generation_events << GenerationEvent.new(*values)
+      end
+
+      def record_discovery_failure(file, reason:)
+        @discovery_failures << DiscoveryFailure.new(file: file.to_s.dup.freeze, reason: reason.to_s.dup.freeze)
+      end
+
+      # A mutant the hazard guard refused to generate (issue #102): counted
+      # for the summary and recorded like any other generation event.
+      def record_hazard_suppression(subject, mutation)
+        @hazard_suppressions += 1
+        record_generation(subject, source: "hazard_guard", operator: mutation.operator, reason: "hazardous")
+      end
+
+      def record_inline_suppression(subject, mutation, directive)
+        @inline_suppressions << {
+          file: subject.file, line: directive.line, subject: subject.id,
+          operator: mutation.operator, reason: directive.reason,
+          digest: mutation.digest(subject_start: subject.start_byte, original_source: subject.file_source)
+        }
+      end
+
+      def record_omission(subject, count:, limit:)
+        @omissions << Omission.new(subject: subject.id.dup.freeze, count: count, limit: limit)
+      end
+
       def inform(message)
         @err.puts("saboteur: #{message}")
       end
 
+      # A heartbeat for a long phase: the supervisor's wait loop calls tick
+      # in the parent after the fork, so no extra thread is ever alive at
+      # fork time. Prints "<label>: N s elapsed" about every interval
+      # seconds, so a long phase does not look stuck.
+      def progress_heartbeat(label, interval: 30)
+        ProgressHeartbeat.new(self, label, interval: interval)
+      end
+
+      # Call before the workers fork. total is the number of mutants the run
+      # may execute; mutants that are skipped or reused never report progress,
+      # so the counter can end below total and the ETA errs on the long side.
+      # The count lives in one file the workers inherit, so every worker adds
+      # to the same number.
+      def start_progress(total)
+        @progress_total = total
+        @progress_started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        @progress_file = Tempfile.new("saboteur-progress")
+        @progress_file.binmode
+        @progress_file.sync = true
+        @err.puts("saboteur: progress #{LEGEND}") unless progress_on_terminal?
+      end
+
       def progress(result)
-        @err.print(MARKS.fetch(result.status, "?"))
+        mark = MARKS.fetch(result.status, "?")
+        return plain_progress(mark) unless @progress_file
+
+        done = record_progress(mark)
+        progress_on_terminal? ? terminal_progress(done) : ci_progress(mark, done)
+        @err.flush
+      end
+
+      def finish_progress
+        @err.puts if @progress_file && (progress_on_terminal? || @progress_file.size % LINE_WIDTH != 0)
+        @progress_file&.close!
+        @progress_file = nil
+      end
+
+      def progress_summary(done)
+        return "#{done}/#{@progress_total}" unless @progress_total.to_i.positive?
+
+        percent = (done * 100.0 / @progress_total).floor
+        "#{done}/#{@progress_total} (#{percent}%)#{eta_text(done)}"
+      end
+
+      private
+
+      def plain_progress(mark)
+        @err.print(mark)
         @err.flush
       end
+
+      def record_progress(mark)
+        @progress_file.write(mark)
+        @progress_file.size
+      end
+
+      def progress_on_terminal?
+        @err.respond_to?(:tty?) && @err.tty?
+      end
+
+      def terminal_progress(done)
+        @err.print("\r\e[Ksaboteur: #{progress_summary(done)}")
+      end
+
+      def ci_progress(mark, done)
+        @err.print(mark)
+        @err.print(" #{progress_summary(done)}\n") if (done % LINE_WIDTH).zero?
+      end
+
+      def eta_text(done)
+        remaining = @progress_total - done
+        return "" if remaining <= 0 || done.zero?
+
+        elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @progress_started
+        seconds = (elapsed / done * remaining).round
+        " eta #{seconds / 60}m#{format('%02d', seconds % 60)}s"
+      end
     end
   end
 end

data/lib/saboteur/engine/diff_scope.rb

@@ -20,6 +20,24 @@ module Saboteur
         end
       end
 
+      # The changed lines a mutant touches, sorted. A statement that spans
+      # several lines is mutated as one piece and reported at its first line,
+      # which can sit above the line that changed; this names the changed line.
+      def changed_lines_for(subject, mutation)
+        changed = @changed_lines.fetch(subject.file, Set.new)
+        mutation.edits.flat_map { |edit| edited_lines(subject, edit).to_a }.uniq.select { changed.include?(_1) }.sort
+      end
+
+      # Distance to the nearest changed line in this file. Unknown outside
+      # the diff is nil, not a made-up nearby location.
+      def distance_for(subject, mutation)
+        changed = @changed_lines.fetch(subject.file, Set.new)
+        mutation.edits.map do |edit|
+          first, last = edited_lines(subject, edit).minmax
+          changed.map { |line| (line - line.clamp(first, last)).abs }.min
+        end.min
+      end
+
       private
 
       def edited_lines(subject, edit)

data/lib/saboteur/engine/distribution.rb

@@ -8,19 +8,37 @@ module Saboteur
     # plain data; the parent rebuilds them against its own copy of the same
     # candidate list, so results look the same either way.
     class Distribution
-      def initialize(jobs:, scheduler:, diagnostics:, cache:, on_start: nil)
+      # After run: the subjects a stop_after callable left unrun (always empty
+      # without one).
+      attr_reader :unreached
+
+      # stop_after: nil, or a callable that gets one subject's results and
+      # returns true to stop before the next subject (--fail-fast). It only
+      # works on one process: parallel workers have already started the
+      # other subjects.
+      def initialize(jobs:, scheduler:, diagnostics:, cache:, on_start: nil, after_subject: nil,
+                     stop_after: nil)
         @jobs = jobs
+        @stop_after = stop_after
+        @unreached = []
         @on_start = on_start
         @scheduler = scheduler
         @diagnostics = diagnostics
         @cache = cache
+        @after_subject = after_subject
       end
 
       # lists: each subject's [[mutation, text]] candidates, in subject order.
       def run(subjects, lists)
-        return subjects.zip(lists).flat_map { |subject, list| @scheduler.call(subject, list) } if @jobs <= 1
+        return run_serially(subjects, lists) if @jobs <= 1
 
-        pool = Execution::WorkerPool.new(jobs: @jobs, on_start: @on_start)
+        # The watcher runs as each subject's row arrives, before the pool
+        # hands out more work, so a tripped canary stops the queue (jobs>1
+        # used to check only after the whole batch returned, issue #102).
+        arrival = lambda do |i, row|
+          @after_subject&.call(subjects[i], decode(row, subjects[i], lists[i]))
+        end
+        pool = Execution::WorkerPool.new(jobs: @jobs, on_start: @on_start, on_result: arrival)
         rows = pool.map(subjects.each_index.to_a, weights: lists.map(&:size)) do |i|
           work(subjects[i], lists[i])
         end
@@ -29,6 +47,20 @@ module Saboteur
 
       private
 
+      def run_serially(subjects, lists)
+        results = []
+        subjects.zip(lists).each_with_index do |(subject, list), index|
+          subject_results = @scheduler.call(subject, list)
+          results.concat(subject_results)
+          @after_subject&.call(subject, subject_results)
+          next unless @stop_after&.call(subject_results)
+
+          @unreached = subjects.drop(index + 1)
+          break
+        end
+        results
+      end
+
       # Runs in a worker.
       def work(subject, list)
         before = @diagnostics.warnings.size
@@ -43,29 +75,38 @@ module Saboteur
           "status" => result.status.to_s, "reason" => result.reason, "runtime" => result.runtime,
           "killed_by" => result.killed_by, "covered_by" => result.covered_by,
           "kill_cause" => result.kill_cause, "reused" => result.reused?,
-          "rejected_kill" => result.rejected_kill?
+          "rejected_kill" => result.rejected_kill?, "executed" => result.executed
         }
       end
 
       def rebuild(row, subject, list)
-        return failed(row["error"], subject, list) if row["error"]
+        if row["error"]
+          @diagnostics.warn("#{subject.id}: #{row['error']}")
+          return failed(row["error"], subject, list)
+        end
 
         @diagnostics.warnings.concat(row["warnings"])
+        decode(row, subject, list).each { |result| @cache&.record(result) unless result.reused? }
+      end
+
+      # Turns a worker row back into results, without diagnostics or caching:
+      # used both at result arrival (the canary) and at rebuild time.
+      def decode(row, subject, list)
+        return failed(row["error"], subject, list) if row["error"]
+
         row["results"].map do |data|
           mutation, text = list.fetch(data["position"])
-          result = MutantResult.new(subject: subject, mutation: mutation, status: data["status"].to_sym,
-                                    mutated_source: text, reason: data["reason"], runtime: data["runtime"],
-                                    killed_by: data["killed_by"], covered_by: data["covered_by"],
-                                    kill_cause: data["kill_cause"], reused: data["reused"],
-                                    rejected_kill: data.fetch("rejected_kill", false))
-          @cache&.record(result) unless result.reused?
-          result
+          MutantResult.new(subject: subject, mutation: mutation, status: data["status"].to_sym,
+                           mutated_source: text, reason: data["reason"], runtime: data["runtime"],
+                           killed_by: data["killed_by"], covered_by: data["covered_by"],
+                           kill_cause: data["kill_cause"], reused: data["reused"],
+                           executed: data["executed"],
+                           rejected_kill: data.fetch("rejected_kill", false))
         end
       end
 
       # The worker died (or raised) while running this subject.
       def failed(message, subject, list)
-        @diagnostics.warn("#{subject.id}: #{message}")
         list.map do |mutation, text|
           MutantResult.new(subject: subject, mutation: mutation, status: :error, mutated_source: text,
                            reason: message)

data/lib/saboteur/engine/result_cache.rb

@@ -11,13 +11,16 @@ module Saboteur
     # Stryker's history files.
     #
     # A verdict is reused only while everything it was computed from is the
-    # same: the method's source, the mutant, the set of covering tests, the
-    # content of those tests' files, the lockfile, the run's inputs (config,
-    # saboteur version). Only killed, survived and timeout are stored;
+    # same: the method's source, the whole file the method lives in (so a
+    # constant or helper changed elsewhere in that file is seen), the mutant,
+    # the set of covering tests, the content of those tests' files, the
+    # lockfile, the run's inputs (config, saboteur version). Only killed, survived
+    # and timeout are stored;
     # errors and skips always run again.
     #
-    # Known hole, the reason this is opt-in: a change in a method this one
-    # calls is not seen, so a stored verdict may be stale after it.
+    # Known hole, the reason this is opt-in: a change in another file that
+    # this method uses (a method it calls, a constant defined elsewhere) is
+    # not seen, so a stored verdict may be stale after it.
     class ResultCache
       FORMAT = 1
       STORED = %i[killed survived timeout].freeze
@@ -75,6 +78,8 @@ module Saboteur
       def fingerprint(subject, tests)
         digest = Digest::SHA256.new
         digest << VERSION << "\0" << @inputs << "\0" << @suite_digest << "\0" << subject.source << "\0"
+        # A hex digest has a fixed width, so it needs no separator after it.
+        digest << Digest::SHA256.hexdigest(subject.file_source)
         tests.sort.each { |test| digest << test << "\0" << file_digest(test_file(test)) << "\0" }
         LOCKFILES.each { |name| digest << file_digest(File.join(@root, name)) << "\0" }
         digest.hexdigest