churn_vs_complexity 1.3.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de43228c387df735f0bd01eff8f53cfac5bf752474be0f4e901d29d5847781f5
-  data.tar.gz: d78475fd751dff01f1aafb45e995cdeabd2d0561348e86de5cf5f951dcc7b6f0
+  metadata.gz: 7484ff3a1c015738808226a78087017f6b7aff5ce42d15879023f32df5648717
+  data.tar.gz: ad3bdeff5ba32e9d7f414b45173d8928b1c463313557202b96aaf5fdcf059109
 SHA512:
-  metadata.gz: 4297b81854c54d4b60672f89156977299549cc012519b178ce206f3108e3e7dc65b9a9d0341b0f218945b9be580f9cc46fb8b5d049a3e5508f4fb4d026e23f5f
-  data.tar.gz: 1338de4b39496a4af2f9485c14178112558399ab308b03d76ad18294e8a5cb6f87f55f5a0990ea0963ae63d3e6a5c49758c5aed072390e2610e2f0e5b52c2723
+  metadata.gz: a56e26296acfff22e755c414cab9ee36923e4ccd7181d67691f55feb8f99b5aa738f5d2b7645006f25c632efa346fb535d7fe04923443dedf607772ce1a21323
+  data.tar.gz: f406ee696facf7708e792b67ed8b463f1554365b3939deb4bde40c9fab88dfe891997f611727559a283103c5b7234f71d78d5fe108731628cb3fd8a782a73ce5

data/CHANGELOG.md

@@ -17,3 +17,9 @@
 
 - Add support for javascript and typescript complexity calculation using eslint
 - Fixed behavior when --since or short-hand flags were not provided
+
+## [1.4.0] - 2024-10-10
+
+- Add timetravel mode to visualise code quality over time
+- Add alpha, beta, and gamma scores to summaries
+- Fixed broken Ruby complexity calculation

data/README.md

@@ -4,7 +4,7 @@ A tool to visualise code complexity in a project and help direct refactoring eff
 
 Inspired by [Michael Feathers' article "Getting Empirical about Refactoring"](https://www.agileconnection.com/article/getting-empirical-about-refactoring) and the gem [turbulence](https://rubygems.org/gems/turbulence) by Chad Fowler and others.
 
-This gem was built primarily to support analysis of Java and Ruby repositories, but it can easily be extended.
+This gem currently supports analysis of Java, Ruby, JavaScript, and TypeScript repositories, but it can easily be extended.
 
 ## Installation
 
@@ -40,14 +40,27 @@ Usage: churn_vs_complexity [options] folder
         --graph                      Format output as HTML page with Churn vs Complexity graph
         --summary                    Output summary statistics (mean and median) for churn and complexity
         --excluded PATTERN           Exclude file paths including this string. Can be used multiple times.
-        --since YYYY-MM-DD           Calculate churn after this date
+        --since YYYY-MM-DD           Normal mode: Calculate churn after this date. Timetravel mode: calculate summaries from this date
     -m, --month                      Calculate churn for the month leading up to the most recent commit
     -q, --quarter                    Calculate churn for the quarter leading up to the most recent commit
     -y, --year                       Calculate churn for the year leading up to the most recent commit
+        --timetravel N               Calculate summary for all commits at intervals of N days throughout project history or from the date specified with --since
         --dry-run                    Echo the chosen options from the CLI
     -h, --help                       Display help
+
+
 ```
 
+Note that when using the `--timetravel` mode, the semantics of some flags are subtly different from normal mode:
+
+* `--since YYYY-MM-DD`: Calculate summaries from this date
+* `--month`, `--quarter`, `--year`: Calculate churn for the period leading up to each commit being summarised
+
+Timetravel analysis can take many minutes for old and large repositories.
+
+Summaries in normal mode include a gamma score, which is an unnormalised harmonic mean of churn and complexity. This allows for comparison of summaries across different projects with the same language, or over time for a single project.
+
+Summary points in timetravel mode instead include an alpha score, which is the same harmonic mean of churn and complexity, where churn and complexity values are normalised to a 0-1 range to avoid either churn or complexity dominating the score. The summary points also include a beta score, which is the geometric mean of the normalised churn and complexity values.
 ## Examples
 
 `churn_vs_complexity --ruby --csv my_ruby_project > ~/Desktop/ruby-demo.csv`
@@ -56,6 +69,7 @@ Usage: churn_vs_complexity [options] folder
 
 `churn_vs_complexity --ruby --summary -m my_ruby_project >> ~/Desktop/monthly-report.txt`
 
+`churn_vs_complexity --java -m --since 2019-03-01 --timetravel 30 --graph my_java_project > ~/Desktop/timetravel-after-1st-march-2019.html`
 
 ## Development
 

data/TODO

@@ -0,0 +1,11 @@
+TODO:
+
+- Move Timetravel calculations away from serializer
+
+- Database, where we can prepopulate the state of every file and every commit
+  - Populate incrementally from each commit in log. 
+  - Only need to care about deltas between commits, and copy everything else from previous commit.
+  - processed_commit table with sha and version of processing logic
+- Unit tests for simpler new classes
+- Integration test for Timetravel
+- Candlebars on mean dots in graph

data/bin/churn_vs_complexity

@@ -6,6 +6,11 @@ require 'churn_vs_complexity'
 
 begin
   ChurnVsComplexity::CLI.run!
+rescue ChurnVsComplexity::ValidationError => e
+  warn e.message
+  exit 1
 rescue StandardError => e
+  warn e.backtrace
   warn e.message
+  exit 2
 end

data/lib/churn_vs_complexity/churn.rb

@@ -10,7 +10,7 @@ module ChurnVsComplexity
           git_dir = File.join(folder, '.git')
           earliest_date = [date_of_first_commit(folder:), since].max
           formatted_date = earliest_date.strftime('%Y-%m-%d')
-          cmd = %Q(git --git-dir #{git_dir} --work-tree #{folder} log --format="%H" --follow --since="#{formatted_date}" -- #{file} | wc -l)
+          cmd = %(git --git-dir #{git_dir} --work-tree #{folder} log --format="%H" --follow --since="#{formatted_date}" -- #{file} | wc -l)
           `#{cmd}`.to_i
         end
 

data/lib/churn_vs_complexity/cli.rb

@@ -22,7 +22,8 @@ module ChurnVsComplexity
           options[:language] = :ruby
         end
 
-        opts.on('--js', '--ts', '--javascript', '--typescript', 'Check complexity of javascript and typescript files') do
+        opts.on('--js', '--ts', '--javascript', '--typescript',
+                'Check complexity of javascript and typescript files',) do
           options[:language] = :javascript
         end
 
@@ -43,20 +44,27 @@ module ChurnVsComplexity
           options[:excluded] << value
         end
 
-        opts.on('--since YYYY-MM-DD', 'Calculate churn after this date') do |value|
+        opts.on('--since YYYY-MM-DD',
+                'Normal mode: Calculate churn after this date. Timetravel mode: calculate summaries from this date',) do |value|
           options[:since] = value
         end
 
         opts.on('-m', '--month', 'Calculate churn for the month leading up to the most recent commit') do
-          options[:since] = :month
+          options[:relative_period] = :month
         end
 
         opts.on('-q', '--quarter', 'Calculate churn for the quarter leading up to the most recent commit') do
-          options[:since] = :quarter
+          options[:relative_period] = :quarter
         end
 
         opts.on('-y', '--year', 'Calculate churn for the year leading up to the most recent commit') do
-          options[:since] = :year
+          options[:relative_period] = :year
+        end
+
+        opts.on('--timetravel N',
+                'Calculate summary for all commits at intervals of N days throughout project history or from the date specified with --since',) do |value|
+          options[:mode] = :timetravel
+          options[:jump_days] = value.to_i
         end
 
         opts.on('--dry-run', 'Echo the chosen options from the CLI') do
@@ -84,7 +92,11 @@ module ChurnVsComplexity
 
       config.validate!
 
-      puts config.to_engine.check(folder:)
+      if options[:mode] == :timetravel
+        puts config.timetravel.go(folder:)
+      else
+        puts config.to_engine.check(folder:)
+      end
     end
   end
 end

data/lib/churn_vs_complexity/complexity/eslint_calculator.rb

@@ -5,20 +5,18 @@ module ChurnVsComplexity
     module ESLintCalculator
       class << self
         def folder_based? = false
-          
+
         def calculate(files:)
           dir_path = File.join(gem_root, 'tmp', 'eslint-support')
           script_path = File.join(dir_path, 'complexity-calculator.js')
           install_command = "npm install --prefix '#{dir_path}'"
           `#{install_command}`
 
-
           command = "node #{script_path} '#{files.to_json}'"
           complexity = `#{command}`
 
-          if complexity.empty?
-            raise Error, "Failed to calculate complexity"
-          end
+          raise Error, 'Failed to calculate complexity' if complexity.empty?
+
           all = JSON.parse(complexity)
           all.to_h do |abc|
             [abc['file'], abc['complexity']]

data/lib/churn_vs_complexity/complexity/flog_calculator.rb

@@ -9,9 +9,9 @@ module ChurnVsComplexity
         def folder_based? = false
 
         def calculate(files:)
-          flog = Flog.new
-          # TODO: Run this concurrently
+          #  TODO: Run this concurrently
           files.to_h do |file|
+            flog = Flog.new
             flog.flog(file)
             [file, flog.total_score]
           end

data/lib/churn_vs_complexity/complexity/pmd_calculator.rb

@@ -9,7 +9,10 @@ module ChurnVsComplexity
         def folder_based? = true
 
         def calculate(folder:)
-          output = `pmd check -d #{folder} -R #{resolve_ruleset_path} -f json -t #{CONCURRENCY}  --cache #{resolve_cache_path}`
+          cache_path = resolve_cache_path
+          output = `pmd check -d #{folder} -R #{resolve_ruleset_path} -f json -t #{CONCURRENCY}  --cache #{cache_path}`
+          File.delete(cache_path)
+
           Parser.new.parse(output)
         end
 
@@ -29,7 +32,7 @@ module ChurnVsComplexity
         end
 
         def resolve_cache_path
-          File.join(gem_root, 'tmp', 'pmd-support', 'pmd-cache')
+          File.join(gem_root, 'tmp', 'pmd-support', "pmd-cache-#{Process.pid}")
         end
 
         def gem_root

data/lib/churn_vs_complexity/concurrent_calculator.rb

@@ -57,7 +57,7 @@ module ChurnVsComplexity
     def combine_results
       result = {}
       result[:values_by_file] = @complexity_results.keys.each_with_object({}) do |file, acc|
-        # File with complexity score might not have churned in queried period, 
+        # File with complexity score might not have churned in queried period,
         # set zero churn on miss
         acc[file] = [@churn_results[file] || 0, @complexity_results[file]]
       end

data/lib/churn_vs_complexity/config.rb

@@ -7,25 +7,42 @@ module ChurnVsComplexity
       serializer:,
       excluded: [],
       since: nil,
+      relative_period: nil,
       complexity_validator: ComplexityValidator,
-      since_validator: SinceValidator
+      since_validator: SinceValidator,
+      **options
     )
       @language = language
       @serializer = serializer
       @excluded = excluded
       @since = since
+      @relative_period = relative_period
       @complexity_validator = complexity_validator
       @since_validator = since_validator
+      @options = options
     end
 
     def validate!
-      raise Error, "Unsupported language: #{@language}" unless %i[java ruby javascript].include?(@language)
-      raise Error, "Unsupported serializer: #{@serializer}" unless %i[none csv graph summary].include?(@serializer)
+      raise ValidationError, "Unsupported language: #{@language}" unless %i[java ruby javascript].include?(@language)
 
-      @since_validator.validate!(@since)
+      SerializerValidator.validate!(serializer: @serializer, mode: @options[:mode])
+
+      @since_validator.validate!(since: @since, relative_period: @relative_period, mode: @options[:mode])
+      RelativePeriodValidator.validate!(relative_period: @relative_period, mode: @options[:mode])
       @complexity_validator.validate!(@language)
     end
 
+    def timetravel
+      engine = timetravel_engine_config.to_engine
+      Timetravel::Traveller.new(
+        since: @since,
+        relative_period: @relative_period,
+        engine:,
+        jump_days: @options[:jump_days],
+        serializer: @serializer,
+      )
+    end
+
     def to_engine
       case @language
       when :java
@@ -34,7 +51,7 @@ module ChurnVsComplexity
           churn:,
           file_selector: FileSelector::Java.excluding(@excluded),
           serializer:,
-          since: @since,
+          since: @since || @relative_period,
         )
       when :ruby
         Engine.concurrent(
@@ -42,7 +59,7 @@ module ChurnVsComplexity
           churn:,
           file_selector: FileSelector::Ruby.excluding(@excluded),
           serializer:,
-          since: @since,
+          since: @since || @relative_period,
         )
       when :javascript
         Engine.concurrent(
@@ -50,13 +67,26 @@ module ChurnVsComplexity
           churn:,
           file_selector: FileSelector::JavaScript.excluding(@excluded),
           serializer:,
-          since: @since,
+          since: @since || @relative_period,
         )
       end
     end
 
     private
 
+    def timetravel_engine_config
+      Config.new(
+        language: @language,
+        serializer: :pass_through,
+        excluded: @excluded,
+        since: nil, # since has a different meaning in timetravel mode
+        relative_period: @relative_period,
+        complexity_validator: @complexity_validator,
+        since_validator: @since_validator,
+        **@options,
+      )
+    end
+
     def churn = Churn::GitCalculator
 
     def serializer
@@ -69,6 +99,8 @@ module ChurnVsComplexity
         Serializer::Graph.new
       when :summary
         Serializer::Summary
+      when :pass_through
+        Serializer::PassThrough
       end
     end
 
@@ -81,21 +113,45 @@ module ChurnVsComplexity
       end
     end
 
+    # TODO: unit test
+    module SerializerValidator
+      def self.validate!(serializer:, mode:)
+        raise ValidationError, "Unsupported serializer: #{serializer}" \
+          unless %i[none csv graph summary].include?(serializer)
+        raise ValidationError, 'Does not support --summary in --timetravel mode' \
+         if serializer == :summary && mode == :timetravel
+      end
+    end
+
+    # TODO: unit test
+    module RelativePeriodValidator
+      def self.validate!(relative_period:, mode:)
+        if mode == :timetravel && relative_period.nil?
+          raise ValidationError,
+                'Relative period is required in timetravel mode'
+        end
+        return if relative_period.nil? || %i[month quarter year].include?(relative_period)
+
+        raise ValidationError, "Invalid relative period #{relative_period}"
+      end
+    end
+
     module SinceValidator
-      def self.validate!(since)
+      def self.validate!(since:, relative_period:, mode:)
         # since can be nil, a date string or a keyword (:month, :quarter, :year)
         return if since.nil?
 
-        if since.is_a?(Symbol)
-          raise Error, "Invalid since value #{since}" unless %i[month quarter year].include?(since)
-        elsif since.is_a?(String)
-          begin
-            Date.strptime(since, '%Y-%m-%d')
-          rescue StandardError
-            raise Error, "Invalid date #{since}, please use correct format, YYYY-MM-DD"
-          end
-        else
-          raise Error, "Invalid since value #{since}"
+        unless mode == :timetravel || since.nil? || relative_period.nil?
+          raise ValidationError,
+                '--since and relative period (--month, --quarter, --year) can only be used together in --timetravel mode'
+        end
+
+        raise ValidationError, "Invalid since value #{since}" unless since.is_a?(String)
+
+        begin
+          Date.strptime(since, '%Y-%m-%d')
+        rescue Date::Error
+          raise ValidationError, "Invalid date #{since}, please use correct format, YYYY-MM-DD"
         end
       end
     end

data/lib/churn_vs_complexity/git_date.rb

@@ -2,6 +2,13 @@
 
 module ChurnVsComplexity
   module GitDate
+    def self.select_dates_with_at_least_interval(dates, interval)
+      ds = dates.sort
+      ds.each_with_object([]) do |date, acc|
+        acc << date if acc.empty? || date - acc.last >= interval
+      end
+    end
+
     def self.git_period(cli_arg_since, latest_commit_date)
       latest_commit_date = latest_commit_date.to_date
       if cli_arg_since.nil?

data/lib/churn_vs_complexity/serializer/csv.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module ChurnVsComplexity
+  module Serializer
+    module CSV
+      def self.serialize(result)
+        values_by_file = result[:values_by_file]
+        values_by_file.map do |file, values|
+          "#{file},#{values[0]},#{values[1]}\n"
+        end.join
+      end
+    end
+  end
+end

data/lib/churn_vs_complexity/serializer/graph.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module ChurnVsComplexity
+  module Serializer
+    class Graph
+      def initialize(template: Graph.load_template_file)
+        @template = template
+      end
+
+      def serialize(result)
+        data = result[:values_by_file].map do |file, values|
+          "{ file_path: '#{file}', churn: #{values[0]}, complexity: #{values[1]} }"
+        end.join(",\n") + "\n"
+        title = Serializer.title(result)
+        @template.gsub("// INSERT DATA\n", data).gsub('INSERT TITLE', title)
+      end
+
+      def self.load_template_file
+        file_path = File.expand_path('../../../tmp/template/graph.html', __dir__)
+        File.read(file_path)
+      end
+    end
+  end
+end

data/lib/churn_vs_complexity/serializer/pass_through.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module ChurnVsComplexity
+  module Serializer
+    module PassThrough
+      class << self
+        def serialize(result)
+          values_by_file = result[:values_by_file]
+          end_date = result[:git_period].end_date
+          values = values_by_file.map do |_, values|
+            [values[0].to_f, values[1].to_f]
+          end
+          {
+            end_date:,
+            values:,
+          }
+        end
+      end
+    end
+  end
+end

data/lib/churn_vs_complexity/serializer/summary.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module ChurnVsComplexity
+  module Serializer
+    module Summary
+      def self.serialize(result)
+        values_by_file = result[:values_by_file]
+        summary = SummaryHash.serialize(result)
+
+        <<~SUMMARY
+          #{Serializer.title(result)}
+
+          Number of observations: #{values_by_file.size}
+
+          Churn:
+          Mean #{summary[:mean_churn]}, Median #{summary[:median_churn]}
+
+          Complexity:
+          Mean #{summary[:mean_complexity]}, Median #{summary[:median_complexity]}
+
+          Gamma score:
+          Mean #{summary[:mean_gamma_score]}, Median #{summary[:median_gamma_score]}
+        SUMMARY
+      end
+    end
+  end
+end

data/lib/churn_vs_complexity/serializer/summary_hash.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module ChurnVsComplexity
+  module Serializer
+    module SummaryHash
+      class << self
+        def serialize(result)
+          values_by_file = result[:values_by_file]
+          churn_values = values_by_file.map { |_, values| values[0].to_f }
+          complexity_values = values_by_file.map { |_, values| values[1].to_f }
+
+          mean_churn = churn_values.sum / churn_values.size
+          median_churn = churn_values.sort[churn_values.size / 2]
+          mean_complexity = complexity_values.sum / complexity_values.size
+          median_complexity = complexity_values.sort[complexity_values.size / 2]
+
+          max_churn = churn_values.max
+          min_churn = churn_values.min
+          max_complexity = complexity_values.max
+          min_complexity = complexity_values.min
+
+          epsilon = 0.0001
+          gamma_score = values_by_file.map do |_, values|
+            # unnormalised harmonic mean of churn and complexity,
+            # since the summary needs to be comparable over time
+            churn = values[0].to_f + epsilon
+            complexity = values[1].to_f + epsilon
+
+            (2 * churn * complexity) / (churn + complexity)
+          end
+
+          mean_gamma_score = gamma_score.sum / gamma_score.size
+          median_gamma_score = gamma_score.sort[gamma_score.size / 2]
+
+          end_date = result[:git_period].end_date
+
+          {
+            mean_churn:,
+            median_churn:,
+            max_churn:,
+            min_churn:,
+            mean_complexity:,
+            median_complexity:,
+            max_complexity:,
+            min_complexity:,
+            mean_gamma_score:,
+            median_gamma_score:,
+            end_date:,
+          }
+        end
+      end
+    end
+  end
+end

data/lib/churn_vs_complexity/serializer/timetravel/quality_calculator.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module ChurnVsComplexity
+  module Serializer
+    module Timetravel
+      EPSILON = 0.0001
+
+      class QualityCalculator
+        def initialize(min_churn:, max_churn:, min_complexity:, max_complexity:)
+          @min_churn = min_churn
+          @max_churn = max_churn
+          @min_complexity = min_complexity
+          @max_complexity = max_complexity
+        end
+
+        def alpha_score(raw_churn, raw_complexity)
+          # harmonic mean of normalised churn and complexity
+          churn = normalise(raw_churn, @min_churn, @max_churn, EPSILON)
+          complexity = normalise(raw_complexity, @min_complexity, @max_complexity, EPSILON)
+
+          (2 * churn * complexity) / (churn + complexity)
+        end
+
+        def beta_score(raw_churn, raw_complexity)
+          # geometric mean of normalised churn and complexity
+          churn = normalise(raw_churn, @min_churn, @max_churn, EPSILON)
+          complexity = normalise(raw_complexity, @min_complexity, @max_complexity, EPSILON)
+
+          Math.sqrt(churn * complexity)
+        end
+
+        private
+
+        def normalise(score, min, max, epsilon) = (score + epsilon - min) / (epsilon + max - min)
+      end
+    end
+  end
+end

data/lib/churn_vs_complexity/serializer/timetravel/stats_calculator.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module ChurnVsComplexity
+  module Serializer
+    module Timetravel
+      class StatsCalculator
+        # ['some_sha', { 'end_date' => '2024-01-01', 'values' => [[1, 2], [3, 4]] }]
+        def summaries(result)
+          observations = result.sort_by do |_sha, summary|
+            summary['end_date']
+          end.map { |entry| entry[1] }
+
+          quality_calculator = QualityCalculator.new(**extrema(observations))
+          observations.map do |o|
+            end_date = o['end_date']
+            scores = o['values'].map do |(churn, complexity)|
+              alpha = quality_calculator.alpha_score(churn, complexity)
+              beta = quality_calculator.beta_score(churn, complexity)
+              [churn, complexity, alpha, beta]
+            end
+            {
+              'end_date' => end_date,
+              'mean_churn' => mean(scores.map { |s| s[0] }),
+              'median_churn' => median(scores.map { |s| s[0] }),
+              'mean_complexity' => mean(scores.map { |s| s[1] }),
+              'median_complexity' => median(scores.map { |s| s[1] }),
+              'mean_alpha_score' => mean(scores.map { |s| s[2] }),
+              'median_alpha_score' => median(scores.map { |s| s[2] }),
+              'mean_beta_score' => mean(scores.map { |s| s[3] }),
+              'median_beta_score' => median(scores.map { |s| s[3] }),
+            }
+          end
+        end
+
+        private
+
+        def extrema(observations)
+          churn_series = observations.flat_map { |o| o['values'] }.map { |(churn, _)| churn }
+          max_churn = churn_series.max
+          min_churn = churn_series.min
+
+          complexity_series = observations.flat_map { |o| o['values'] }.map { |(_, complexity)| complexity }
+          max_complexity = complexity_series.max
+          min_complexity = complexity_series.min
+
+          { max_churn:, min_churn:, max_complexity:, min_complexity: }
+        end
+
+        def mean(series)
+          series.sum / series.size
+        end
+
+        def median(series)
+          sorted = series.sort
+          sorted[sorted.size / 2]
+        end
+      end
+    end
+  end
+end

data/lib/churn_vs_complexity/serializer/timetravel.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require_relative 'timetravel/quality_calculator'
+require_relative 'timetravel/stats_calculator'
+
+module ChurnVsComplexity
+  module Serializer
+    module Timetravel
+      def self.summaries(result)
+        StatsCalculator.new.summaries(result)
+      end
+
+      def self.resolve(serializer:, git_period:, relative_period:, jump_days:)
+        case serializer
+        when :csv
+          CSV
+        when :graph
+          Graph.new(git_period:, relative_period:, jump_days:)
+        end
+      end
+
+      module CSV
+        def self.serialize(result)
+          summaries = Timetravel.summaries(result)
+
+          # 2. Add title row to front of summaries
+          summaries.unshift(
+            {
+              'end_date' => 'Date',
+              'mean_churn' => 'Mean Churn',
+              'median_churn' => 'Median Churn',
+              'mean_complexity' => 'Mean Complexity',
+              'median_complexity' => 'Median Complexity',
+              'mean_alpha_score' => 'Mean Alpha Score',
+              'median_alpha_score' => 'Median Alpha Score',
+              'mean_beta_score' => 'Mean Beta Score',
+              'median_beta_score' => 'Median Beta Score',
+            },
+          )
+
+          # 3. convert to csv
+          summaries.map do |summary|
+            "#{summary['end_date']},#{summary['mean_churn']},#{summary['median_churn']},#{summary['mean_complexity']},#{summary['median_complexity']},#{summary['mean_alpha_score']},#{summary['median_alpha_score']},#{summary['mean_beta_score']},#{summary['median_beta_score']}"
+          end.join("\n")
+        end
+      end
+
+      # TODO: unit test
+      class Graph
+        def initialize(git_period:, relative_period:, jump_days:, template: Graph.load_template_file)
+          @template = template
+          @git_period = git_period
+          @relative_period = relative_period
+          @jump_days = jump_days
+        end
+
+        def self.load_template_file
+          file_path = File.expand_path('../../../tmp/template/timetravel_graph.html', __dir__)
+          File.read(file_path)
+        end
+
+        def serialize(result)
+          summaries = Timetravel.summaries(result)
+
+          data = summaries.map do |summary|
+            JSON.dump(summary)
+          end.join(",\n") + "\n"
+
+          @template.gsub("// INSERT DATA\n", data)
+                   .gsub('INSERT TITLE', title)
+                   .gsub('INSERT CHURN MODIFIER', churn_modifier)
+        end
+
+        private
+
+        def title
+          "#{churn_modifier}churn and complexity since #{since} evaluated every #{@jump_days} days"
+        end
+
+        def since
+          if @git_period.requested_start_date.nil?
+            'start of project'
+          else
+            @git_period.effective_start_date.strftime('%Y-%m-%d').to_s
+          end
+        end
+
+        def churn_modifier
+          case @relative_period
+          when :month
+            'Monthly '
+          when :quarter
+            'Quarterly '
+          when :year
+            'Yearly '
+          else
+            ''
+          end
+        end
+      end
+    end
+  end
+end

data/lib/churn_vs_complexity/serializer.rb

@@ -1,5 +1,12 @@
 # frozen_string_literal: true
 
+require_relative 'serializer/timetravel'
+require_relative 'serializer/summary_hash'
+require_relative 'serializer/summary'
+require_relative 'serializer/csv'
+require_relative 'serializer/graph'
+require_relative 'serializer/pass_through'
+
 module ChurnVsComplexity
   module Serializer
     def self.title(result)
@@ -15,65 +22,5 @@ module ChurnVsComplexity
     module None
       def self.serialize(result) = result
     end
-
-    module Summary
-      def self.serialize(result)
-        values_by_file = result[:values_by_file]
-        churn_values = values_by_file.map { |_, values| values[0].to_f }
-        complexity_values = values_by_file.map { |_, values| values[1].to_f }
-
-        mean_churn = churn_values.sum / churn_values.size
-        median_churn = churn_values.sort[churn_values.size / 2]
-        mean_complexity = complexity_values.sum / complexity_values.size
-        median_complexity = complexity_values.sort[complexity_values.size / 2]
-
-        product = values_by_file.map { |_, values| values[0].to_f * values[1].to_f }
-        mean_product = product.sum / product.size
-        median_product = product.sort[product.size / 2]
-
-        <<~SUMMARY
-          #{Serializer.title(result)}
-
-          Number of observations: #{values_by_file.size}
-
-          Churn:
-          Mean #{mean_churn}, Median #{median_churn}
-
-          Complexity:
-          Mean #{mean_complexity}, Median #{median_complexity}
-
-          Product of churn and complexity:
-          Mean #{mean_product}, Median #{median_product}
-        SUMMARY
-      end
-    end
-
-    module CSV
-      def self.serialize(result)
-        values_by_file = result[:values_by_file]
-        values_by_file.map do |file, values|
-          "#{file},#{values[0]},#{values[1]}\n"
-        end.join
-      end
-    end
-
-    class Graph
-      def initialize(template: Graph.load_template_file)
-        @template = template
-      end
-
-      def serialize(result)
-        data = result[:values_by_file].map do |file, values|
-          "{ file_path: '#{file}', churn: #{values[0]}, complexity: #{values[1]} }"
-        end.join(",\n") + "\n"
-        title = Serializer.title(result)
-        @template.gsub("// INSERT DATA\n", data).gsub('INSERT TITLE', title)
-      end
-
-      def self.load_template_file
-        file_path = File.expand_path('../../tmp/template/graph.html', __dir__)
-        File.read(file_path)
-      end
-    end
   end
 end

data/lib/churn_vs_complexity/timetravel/traveller.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module ChurnVsComplexity
+  # TODO: unit test and integration test
+  module Timetravel
+    class Traveller
+      def initialize(since:, relative_period:, engine:, serializer:, jump_days:, factory: Factory)
+        @relative_period = relative_period
+        @engine = engine
+        @jump_days = jump_days
+        @serializer = serializer
+        @git_period = GitDate.git_period(since, Time.now.to_date)
+        @factory = factory
+      end
+
+      def go(folder:)
+        git_strategy = @factory.git_strategy(folder:)
+        commits = git_strategy.resolve_commits_with_interval(git_period: @git_period, jump_days: @jump_days)
+
+        chunked = make_chunks(commits)
+        work_on(chunked:, folder:, git_strategy:)
+        combined = chunked.map { |c_and_p| read_result(c_and_p[:pipe]) }.reduce({}, :merge)
+
+        serializer.serialize(combined)
+      end
+
+      private
+
+      def work_on(chunked:, folder:, git_strategy:)
+        chunked.map.with_index do |c_and_p, i|
+          worktree = @factory.worktree(root_folder: folder, git_strategy:, number: i)
+          worktree.prepare
+          schedule_work(worktree:, **c_and_p)
+        end
+      end
+
+      def make_chunks(commits)
+        chunk_size = (commits.size / 3.0).ceil
+        commits.each_slice(chunk_size).map do |chunk|
+          { chunk:, pipe: @factory.pipe }
+        end.to_a
+      end
+
+      def read_result(pipe)
+        part = begin
+          JSON.parse(pipe[0].gets)
+        rescue StandardError => e
+          warn "Error parsing JSON: #{e}"
+          {}
+        end
+        pipe.each(&:close)
+        part
+      end
+
+      def schedule_work(chunk:, worktree:, pipe:)
+        @factory.worker(engine: @engine, worktree:)
+                .schedule(chunk:, pipe:)
+      end
+
+      def serializer
+        @factory.serializer(serializer: @serializer, git_period: @git_period,
+                            relative_period: @relative_period, jump_days: @jump_days,)
+      end
+    end
+  end
+end

data/lib/churn_vs_complexity/timetravel/worktree.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'tmpdir'
+
+module ChurnVsComplexity
+  module Timetravel
+    class Worktree
+      attr_reader :folder
+
+      def initialize(root_folder:, git_strategy:, number:)
+        @root_folder = root_folder
+        @git_strategy = git_strategy
+        @number = number
+      end
+
+      def prepare
+        @folder = prepare_worktree
+      end
+
+      def checkout(sha)
+        raise Error, 'Worktree not prepared' if @folder.nil?
+
+        @git_strategy.checkout_in_worktree(@folder, sha)
+      end
+
+      def remove
+        raise Error, 'Worktree not prepared' if @folder.nil?
+
+        @git_strategy.remove_worktree(@folder)
+      end
+
+      private
+
+      def tt_folder
+        folder_hash = Digest::SHA256.hexdigest(@root_folder)[0..7]
+        File.join(Dir.tmpdir, 'churn_vs_complexity', 'timetravel', folder_hash)
+      end
+
+      def prepare_worktree
+        worktree_folder = File.join(tt_folder, "worktree_#{@number}")
+
+        unless File.directory?(worktree_folder)
+          begin
+            FileUtils.mkdir_p(worktree_folder)
+          rescue StandardError
+            nil
+          end
+          @git_strategy.add_worktree(worktree_folder)
+        end
+
+        worktree_folder
+      end
+    end
+  end
+end

data/lib/churn_vs_complexity/timetravel.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require_relative 'timetravel/traveller'
+require_relative 'timetravel/worktree'
+
+module ChurnVsComplexity
+  module Timetravel
+    class Factory
+      def self.git_strategy(folder:) = GitStrategy.new(folder:)
+      def self.pipe = IO.pipe
+      def self.worker(engine:, worktree:) = Worker.new(engine:, worktree:)
+      def self.worktree(root_folder:, git_strategy:, number:) = Worktree.new(root_folder:, git_strategy:, number:)
+      def self.serializer(**args) = Serializer::Timetravel.resolve(**args)
+    end
+
+    class Worker
+      def initialize(engine:, worktree:)
+        @engine = engine
+        @worktree = worktree
+      end
+
+      def schedule(chunk:, pipe:)
+        fork do
+          results = chunk.to_h do |commit|
+            sha = commit.sha
+            @worktree.checkout(sha)
+            result = @engine.check(folder: @worktree.folder)
+            [sha, result]
+          end
+          @worktree.remove
+          pipe[1].puts(JSON.dump(results))
+          pipe[1].close
+        end
+      end
+    end
+
+    class GitStrategy
+      def initialize(folder:)
+        @repo = Git.open(folder)
+        @folder = folder
+      end
+
+      def checkout_in_worktree(worktree_folder, sha)
+        command = "(cd #{worktree_folder} && git checkout #{sha}) > /dev/null 2>&1"
+        `#{command}`
+      end
+
+      def resolve_commits_with_interval(git_period:, jump_days:)
+        candidates = @repo.log(1_000_000).since(git_period.effective_start_date).until(git_period.end_date).to_a
+
+        commits_by_date = candidates.filter { |c| c.date.to_date >= git_period.effective_start_date }
+                                    .group_by { |c| c.date.to_date }
+
+        found_dates = GitDate.select_dates_with_at_least_interval(commits_by_date.keys, jump_days)
+
+        found_dates.map { |date| commits_by_date[date].max_by(&:date) }
+      end
+
+      def add_worktree(wt_folder)
+        command = "(cd #{@folder} && git worktree add -f #{wt_folder}) > /dev/null 2>&1"
+        `#{command}`
+      end
+
+      def remove_worktree(worktree_folder)
+        command = "(cd #{worktree_folder} && git worktree remove -f #{worktree_folder}) > /dev/null 2>&1"
+        `#{command}`
+      end
+    end
+  end
+end

data/lib/churn_vs_complexity/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ChurnVsComplexity
-  VERSION = '1.3.0'
+  VERSION = '1.4.0'
 end

data/lib/churn_vs_complexity.rb

@@ -13,7 +13,9 @@ require_relative 'churn_vs_complexity/cli'
 require_relative 'churn_vs_complexity/config'
 require_relative 'churn_vs_complexity/serializer'
 require_relative 'churn_vs_complexity/git_date'
+require_relative 'churn_vs_complexity/timetravel'
 
 module ChurnVsComplexity
   class Error < StandardError; end
+  class ValidationError < Error; end
 end

data/tmp/template/graph.html

@@ -16,10 +16,7 @@
     ];
 
     // Extract data for Chart.js
-    const labels = dataPoints.map(point => point.file_path);
-    const churnData = dataPoints.map(point => point.churn);
-    const complexityData = dataPoints.map(point => point.complexity);
-
+    const labels = dataPoints.map(point => point.file_path);    
     // Prepare data in Chart.js format
     const data = {
       labels: labels,

data/tmp/template/timetravel_graph.html

@@ -0,0 +1,100 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>INSERT TITLE</title>
+  <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
+  <script src="https://cdn.jsdelivr.net/npm/chartjs-adapter-date-fns"></script>
+  <style>
+    body {
+      font-family: Arial, sans-serif;
+    }
+    h1 {
+      text-align: center;
+      font-size: 24px;
+      font-weight: bold;
+      margin-bottom: 20px;
+      color: #333;
+    }
+    canvas {
+      margin: 20px auto;
+    }
+  </style>
+</head>
+<body>
+  <h1>INSERT TITLE</h1>
+  <canvas id="complexityChart" width="800" height="400"></canvas>
+  <canvas id="churnChart" width="800" height="400"></canvas>
+  <canvas id="alphaScoreChart" width="800" height="400"></canvas>
+  <canvas id="betaScoreChart" width="800" height="400"></canvas>
+
+  <script>
+    
+    const dataPoints = [
+      // INSERT DATA
+    ];
+
+    // Extract dates for x-axis
+    const labels = dataPoints.map(point => point.end_date);
+
+    // Function to create a dataset
+    function createDataset(label, data, color) {
+      return {
+        label: label,
+        data: data,
+        borderColor: color,
+        backgroundColor: color,
+        fill: false,
+        tension: 0.1
+      };
+    }
+
+    // Function to create a chart
+    function createChart(ctx, title, datasets) {
+      return new Chart(ctx, {
+        type: 'line',
+        data: { labels: labels, datasets: datasets },
+        options: {
+          responsive: true,
+          plugins: {
+            title: { display: true, text: title }
+          },
+          scales: {
+            x: { type: 'time', time: { parser: 'yyyy-MM-dd', tooltipFormat: 'll' } },
+            y: { beginAtZero: true }
+          }
+        }
+      });
+    }
+
+    // Create Complexity Chart
+    const complexityCtx = document.getElementById('complexityChart').getContext('2d');
+    createChart(complexityCtx, 'Complexity Over Time', [
+      createDataset('Mean Complexity', dataPoints.map(p => ({ x: p.end_date, y: p.mean_complexity })), 'rgb(75, 192, 192)'),
+      createDataset('Median Complexity', dataPoints.map(p => ({ x: p.end_date, y: p.median_complexity })), 'rgb(255, 99, 132)')
+    ]);
+
+    // Create Churn Chart
+    const churnCtx = document.getElementById('churnChart').getContext('2d');
+    createChart(churnCtx, 'INSERT CHURN MODIFIERChurn Over Time', [
+      createDataset('Mean Churn', dataPoints.map(p => ({ x: p.end_date, y: p.mean_churn })), 'rgb(54, 162, 235)'),
+      createDataset('Median Churn', dataPoints.map(p => ({ x: p.end_date, y: p.median_churn })), 'rgb(255, 206, 86)')
+    ]);
+
+    // Create Alpha Score Chart
+    const alphaScoreCtx = document.getElementById('alphaScoreChart').getContext('2d');
+    createChart(alphaScoreCtx, 'Alpha Score Over Time', [
+      createDataset('Mean Alpha Score', dataPoints.map(p => ({ x: p.end_date, y: p.mean_alpha_score })), 'rgb(153, 102, 255)'),
+      createDataset('Median Alpha Score', dataPoints.map(p => ({ x: p.end_date, y: p.median_alpha_score })), 'rgb(255, 159, 64)')
+    ]);
+
+    // Create Beta Score Chart  
+    const betaScoreCtx = document.getElementById('betaScoreChart').getContext('2d');
+    createChart(betaScoreCtx, 'Beta Score Over Time', [
+      createDataset('Mean Beta Score', dataPoints.map(p => ({ x: p.end_date, y: p.mean_beta_score })), 'rgb(153, 102, 255)'),
+      createDataset('Median Beta Score', dataPoints.map(p => ({ x: p.end_date, y: p.median_beta_score })), 'rgb(255, 159, 64)')
+    ]);
+  </script>
+</body>
+</html>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: churn_vs_complexity
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.0
 platform: ruby
 authors:
 - Erik T. Madsen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-09-25 00:00:00.000000000 Z
+date: 2024-10-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: flog
@@ -57,6 +57,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- TODO
 - bin/churn_vs_complexity
 - lib/churn_vs_complexity.rb
 - lib/churn_vs_complexity/churn.rb
@@ -71,12 +72,24 @@ files:
 - lib/churn_vs_complexity/file_selector.rb
 - lib/churn_vs_complexity/git_date.rb
 - lib/churn_vs_complexity/serializer.rb
+- lib/churn_vs_complexity/serializer/csv.rb
+- lib/churn_vs_complexity/serializer/graph.rb
+- lib/churn_vs_complexity/serializer/pass_through.rb
+- lib/churn_vs_complexity/serializer/summary.rb
+- lib/churn_vs_complexity/serializer/summary_hash.rb
+- lib/churn_vs_complexity/serializer/timetravel.rb
+- lib/churn_vs_complexity/serializer/timetravel/quality_calculator.rb
+- lib/churn_vs_complexity/serializer/timetravel/stats_calculator.rb
+- lib/churn_vs_complexity/timetravel.rb
+- lib/churn_vs_complexity/timetravel/traveller.rb
+- lib/churn_vs_complexity/timetravel/worktree.rb
 - lib/churn_vs_complexity/version.rb
 - package-lock.json
 - tmp/eslint-support/complexity-calculator.js
 - tmp/eslint-support/package.json
 - tmp/pmd-support/ruleset.xml
 - tmp/template/graph.html
+- tmp/template/timetravel_graph.html
 - tmp/test-support/java/small-example/src/main/java/org/example/Main.java
 - tmp/test-support/java/small-example/src/main/java/org/example/spice/Checker.java
 - tmp/test-support/javascript/complex.js
@@ -93,6 +106,7 @@ files:
 - tmp/test-support/txt/st.txt
 - tmp/test-support/txt/uvx.txt
 - tmp/test-support/txt/yz.txt
+- tmp/timetravel/.keep
 homepage: https://github.com/beatmadsen/churn_vs_complexity
 licenses:
 - MIT