csvops 0.4.0.alpha → 0.6.0.alpha

data/lib/csvtool/cli.rb

@@ -2,10 +2,11 @@
 
 require "csv"
 require "csvtool/interface/cli/menu_loop"
-require "csvtool/application/use_cases/run_extraction"
-require "csvtool/application/use_cases/run_row_extraction"
-require "csvtool/application/use_cases/run_row_randomization"
+require "csvtool/interface/cli/workflows/run_extraction_workflow"
+require "csvtool/interface/cli/workflows/run_row_extraction_workflow"
+require "csvtool/interface/cli/workflows/run_row_randomization_workflow"
 require "csvtool/interface/cli/workflows/run_cross_csv_dedupe_workflow"
+require "csvtool/interface/cli/workflows/run_csv_parity_workflow"
 require "csvtool/interface/cli/errors/presenter"
 require "csvtool/infrastructure/csv/header_reader"
 require "csvtool/infrastructure/csv/value_streamer"
@@ -18,6 +19,7 @@ module Csvtool
       "Extract rows (range)",
       "Randomize rows",
       "Dedupe using another CSV",
+      "Validate parity",
       "Exit"
     ].freeze
 
@@ -47,10 +49,11 @@ module Csvtool
     private
 
     def run_menu_loop
-      extract_column_action = -> { Application::UseCases::RunExtraction.new(stdin: @stdin, stdout: @stdout).call }
-      extract_rows_action = -> { Application::UseCases::RunRowExtraction.new(stdin: @stdin, stdout: @stdout).call }
-      randomize_rows_action = -> { Application::UseCases::RunRowRandomization.new(stdin: @stdin, stdout: @stdout).call }
+      extract_column_action = -> { Interface::CLI::Workflows::RunExtractionWorkflow.new(stdin: @stdin, stdout: @stdout).call }
+      extract_rows_action = -> { Interface::CLI::Workflows::RunRowExtractionWorkflow.new(stdin: @stdin, stdout: @stdout).call }
+      randomize_rows_action = -> { Interface::CLI::Workflows::RunRowRandomizationWorkflow.new(stdin: @stdin, stdout: @stdout).call }
       dedupe_action = -> { Interface::CLI::Workflows::RunCrossCsvDedupeWorkflow.new(stdin: @stdin, stdout: @stdout).call }
+      parity_action = -> { Interface::CLI::Workflows::RunCsvParityWorkflow.new(stdin: @stdin, stdout: @stdout).call }
       Interface::CLI::MenuLoop.new(
         stdin: @stdin,
         stdout: @stdout,
@@ -58,7 +61,8 @@ module Csvtool
         extract_column_action: extract_column_action,
         extract_rows_action: extract_rows_action,
         randomize_rows_action: randomize_rows_action,
-        dedupe_action: dedupe_action
+        dedupe_action: dedupe_action,
+        parity_action: parity_action
       ).run
     end
 

data/lib/csvtool/domain/csv_parity_session/parity_options.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Domain
+    module CsvParitySession
+      class ParityOptions
+        attr_reader :separator
+
+        def initialize(separator:, headers_present:)
+          raise ArgumentError, "separator cannot be empty" if separator.to_s.empty?
+
+          @separator = separator
+          @headers_present = headers_present
+        end
+
+        def headers_present?
+          @headers_present
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/domain/csv_parity_session/parity_session.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Domain
+    module CsvParitySession
+      class ParitySession
+        attr_reader :source_pair, :options
+
+        def self.start(source_pair:, options:)
+          new(source_pair: source_pair, options: options)
+        end
+
+        def initialize(source_pair:, options:)
+          @source_pair = source_pair
+          @options = options
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/domain/csv_parity_session/source_pair.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Domain
+    module CsvParitySession
+      class SourcePair
+        attr_reader :left_path, :right_path
+
+        def initialize(left_path:, right_path:)
+          raise ArgumentError, "left_path cannot be empty" if left_path.to_s.empty?
+          raise ArgumentError, "right_path cannot be empty" if right_path.to_s.empty?
+
+          @left_path = left_path
+          @right_path = right_path
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/infrastructure/csv/csv_parity_comparator.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Csvtool
+  module Infrastructure
+    module CSV
+      class CsvParityComparator
+        def call(left_path:, right_path:, col_sep:, headers_present:, sample_limit: 5)
+          deltas = Hash.new(0)
+          left_rows = stream_rows(path: left_path, col_sep: col_sep, headers_present: headers_present) do |key|
+            deltas[key] += 1
+          end
+          right_rows = stream_rows(path: right_path, col_sep: col_sep, headers_present: headers_present) do |key|
+            deltas[key] -= 1
+          end
+
+          left_only_count, right_only_count, left_only_examples, right_only_examples =
+            mismatch_totals_and_samples(deltas: deltas, sample_limit: sample_limit)
+
+          {
+            match: left_only_count.zero? && right_only_count.zero?,
+            left_rows: left_rows,
+            right_rows: right_rows,
+            left_only_count: left_only_count,
+            right_only_count: right_only_count,
+            left_only_examples: left_only_examples,
+            right_only_examples: right_only_examples
+          }
+        end
+
+        private
+
+        def stream_rows(path:, col_sep:, headers_present:)
+          rows = 0
+
+          ::CSV.foreach(path, headers: headers_present, col_sep: col_sep) do |row|
+            fields = headers_present ? row.fields : row
+            yield serialize(fields: fields, col_sep: col_sep)
+            rows += 1
+          end
+
+          rows
+        end
+
+        def mismatch_totals_and_samples(deltas:, sample_limit:)
+          left_only_count = 0
+          right_only_count = 0
+          left_only_examples = []
+          right_only_examples = []
+
+          deltas.each do |key, delta|
+            if delta.positive?
+              left_only_count += delta
+              left_only_examples << { row: key, count_delta: delta } if left_only_examples.length < sample_limit
+            elsif delta.negative?
+              right_only_count += -delta
+              right_only_examples << { row: key, count_delta: -delta } if right_only_examples.length < sample_limit
+            end
+          end
+
+          [left_only_count, right_only_count, left_only_examples, right_only_examples]
+        end
+
+        def serialize(fields:, col_sep:)
+          ::CSV.generate_line(fields, row_sep: "", col_sep: col_sep).chomp
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/infrastructure/output/csv_cross_csv_dedupe_file_writer.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Csvtool
+  module Infrastructure
+    module Output
+      class CsvCrossCsvDedupeFileWriter
+        def initialize(deduper:)
+          @deduper = deduper
+        end
+
+        def call(path:, headers:, col_sep:, dedupe_options:)
+          stats = nil
+          ::CSV.open(path, "w", write_headers: !headers.nil?, headers: headers, col_sep: col_sep) do |csv|
+            stats = @deduper.each_retained(**dedupe_options) { |fields| csv << fields }
+          end
+          stats
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/infrastructure/output/csv_file_writer.rb

@@ -6,9 +6,7 @@ module Csvtool
   module Infrastructure
     module Output
       class CsvFileWriter
-        def initialize(stdout:, errors:, value_streamer:)
-          @stdout = stdout
-          @errors = errors
+        def initialize(value_streamer:)
           @value_streamer = value_streamer
         end
 
@@ -19,10 +17,6 @@ module Csvtool
               csv << [value]
             end
           end
-
-          @stdout.puts "Wrote output to #{output_path}"
-        rescue Errno::EACCES, Errno::ENOENT => e
-          @errors.cannot_write_output_file(output_path, e.class)
         end
       end
     end

data/lib/csvtool/infrastructure/output/csv_randomized_row_file_writer.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Csvtool
+  module Infrastructure
+    module Output
+      class CsvRandomizedRowFileWriter
+        def initialize(row_randomizer:)
+          @row_randomizer = row_randomizer
+        end
+
+        def call(path:, headers:, file_path:, col_sep:, headers_present:, seed:)
+          ::CSV.open(path, "w", write_headers: !headers.nil?, headers: headers, col_sep: col_sep) do |csv|
+            @row_randomizer.each(file_path: file_path, col_sep: col_sep, headers: headers_present, seed: seed) do |fields|
+              csv << fields
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/infrastructure/output/csv_row_file_writer.rb

@@ -6,9 +6,7 @@ module Csvtool
   module Infrastructure
     module Output
       class CsvRowFileWriter
-        def initialize(stdout:, errors:, row_streamer:)
-          @stdout = stdout
-          @errors = errors
+        def initialize(row_streamer:)
           @row_streamer = row_streamer
         end
 
@@ -30,12 +28,7 @@ module Csvtool
             csv << fields
           end
 
-          csv&.close
-          @stdout.puts "Wrote output to #{output_path}" if wrote_rows
-          stats
-        rescue Errno::EACCES, Errno::ENOENT => e
-          @errors.cannot_write_output_file(output_path, e.class)
-          nil
+          stats.merge(wrote_rows: wrote_rows)
         ensure
           csv&.close unless csv&.closed?
         end

data/lib/csvtool/interface/cli/errors/presenter.rb

@@ -72,6 +72,10 @@ module Csvtool
           def row_range_out_of_bounds(total_rows)
             @stdout.puts "Row range is out of bounds. File has #{total_rows} data rows."
           end
+
+          def header_mismatch
+            @stdout.puts "CSV headers do not match."
+          end
         end
       end
     end

data/lib/csvtool/interface/cli/menu_loop.rb

@@ -4,7 +4,7 @@ module Csvtool
   module Interface
     module CLI
       class MenuLoop
-        def initialize(stdin:, stdout:, menu_options:, extract_column_action:, extract_rows_action:, randomize_rows_action:, dedupe_action:)
+        def initialize(stdin:, stdout:, menu_options:, extract_column_action:, extract_rows_action:, randomize_rows_action:, dedupe_action:, parity_action:)
           @stdin = stdin
           @stdout = stdout
           @menu_options = menu_options
@@ -12,6 +12,7 @@ module Csvtool
           @extract_rows_action = extract_rows_action
           @randomize_rows_action = randomize_rows_action
           @dedupe_action = dedupe_action
+          @parity_action = parity_action
         end
 
         def run
@@ -31,9 +32,11 @@ module Csvtool
             when "4"
               @dedupe_action.call
             when "5"
+              @parity_action.call
+            when "6"
               return 0
             else
-              @stdout.puts "Please choose 1, 2, 3, 4, or 5."
+              @stdout.puts "Please choose 1, 2, 3, 4, 5, or 6."
             end
           end
         end

data/lib/csvtool/interface/cli/prompts/dedupe_key_selector_prompt.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "csvtool/domain/cross_csv_dedupe_session/column_selector"
+
+module Csvtool
+  module Interface
+    module CLI
+      module Prompts
+        class DedupeKeySelectorPrompt
+          def initialize(stdin:, stdout:)
+            @stdin = stdin
+            @stdout = stdout
+          end
+
+          def call(label:, headers_present:)
+            if headers_present
+              @stdout.print "#{label} key column name: "
+            else
+              @stdout.print "#{label} key column index (1-based): "
+            end
+            input = @stdin.gets&.strip.to_s
+            Domain::CrossCsvDedupeSession::ColumnSelector.from_input(headers_present: headers_present, input: input)
+          rescue ArgumentError
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/prompts/file_path_prompt.rb

@@ -5,13 +5,15 @@ module Csvtool
     module CLI
       module Prompts
         class FilePathPrompt
+          DEFAULT_LABEL = "CSV file path: "
+
           def initialize(stdin:, stdout:)
             @stdin = stdin
             @stdout = stdout
           end
 
-          def call
-            @stdout.print "CSV file path: "
+          def call(label: DEFAULT_LABEL)
+            @stdout.print label
             @stdin.gets&.strip.to_s
           end
         end

data/lib/csvtool/interface/cli/prompts/headers_present_prompt.rb

@@ -5,13 +5,15 @@ module Csvtool
     module CLI
       module Prompts
         class HeadersPresentPrompt
+          DEFAULT_LABEL = "Headers present? [Y/n]: "
+
           def initialize(stdin:, stdout:)
             @stdin = stdin
             @stdout = stdout
           end
 
-          def call
-            @stdout.print "Headers present? [Y/n]: "
+          def call(label: DEFAULT_LABEL)
+            @stdout.print label
             answer = @stdin.gets&.strip.to_s.downcase
             !%w[n no].include?(answer)
           end

data/lib/csvtool/interface/cli/prompts/separator_prompt.rb

@@ -5,14 +5,16 @@ module Csvtool
     module CLI
       module Prompts
         class SeparatorPrompt
+          DEFAULT_LABEL = "Choose separator:"
+
           def initialize(stdin:, stdout:, errors:)
             @stdin = stdin
             @stdout = stdout
             @errors = errors
           end
 
-          def call
-            @stdout.puts "Choose separator:"
+          def call(label: DEFAULT_LABEL)
+            @stdout.puts label
             @stdout.puts "1. comma (,)"
             @stdout.puts "2. tab (\\t)"
             @stdout.puts "3. semicolon (;)"

data/lib/csvtool/interface/cli/prompts/yes_no_prompt.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Interface
+    module CLI
+      module Prompts
+        class YesNoPrompt
+          def initialize(stdin:, stdout:)
+            @stdin = stdin
+            @stdout = stdout
+          end
+
+          def call(label:, default:)
+            @stdout.print label
+            answer = @stdin.gets&.strip.to_s.downcase
+            return default if answer.empty?
+            return true if %w[y yes].include?(answer)
+            return false if %w[n no].include?(answer)
+
+            default
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/workflows/builders/column_session_builder.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "csvtool/domain/column_session/separator"
+require "csvtool/domain/column_session/csv_source"
+require "csvtool/domain/column_session/column_selection"
+require "csvtool/domain/column_session/extraction_options"
+require "csvtool/domain/column_session/column_session"
+
+module Csvtool
+  module Interface
+    module CLI
+      module Workflows
+        module Builders
+          class ColumnSessionBuilder
+            def call(file_path:, col_sep:, column_name:, skip_blanks:)
+              separator = Domain::ColumnSession::Separator.new(col_sep)
+              source = Domain::ColumnSession::CsvSource.new(path: file_path, separator: separator)
+              column_selection = Domain::ColumnSession::ColumnSelection.new(name: column_name)
+              options = Domain::ColumnSession::ExtractionOptions.new(skip_blanks: skip_blanks, preview_limit: 10)
+
+              Domain::ColumnSession::ColumnSession.start(
+                source: source,
+                column_selection: column_selection,
+                options: options
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/workflows/builders/cross_csv_dedupe_session_builder.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "csvtool/domain/cross_csv_dedupe_session/key_mapping"
+require "csvtool/domain/cross_csv_dedupe_session/match_options"
+require "csvtool/domain/cross_csv_dedupe_session/cross_csv_dedupe_session"
+
+module Csvtool
+  module Interface
+    module CLI
+      module Workflows
+        module Builders
+          class CrossCsvDedupeSessionBuilder
+            def call(source:, reference:, source_selector:, reference_selector:, trim_whitespace:, case_insensitive:, destination:)
+              key_mapping = Domain::CrossCsvDedupeSession::KeyMapping.new(
+                source_selector: source_selector,
+                reference_selector: reference_selector
+              )
+              match_options = Domain::CrossCsvDedupeSession::MatchOptions.new(
+                trim_whitespace: trim_whitespace,
+                case_insensitive: case_insensitive
+              )
+
+              Domain::CrossCsvDedupeSession::CrossCsvDedupeSession.start(
+                source: source,
+                reference: reference,
+                key_mapping: key_mapping,
+                match_options: match_options
+              ).with_output_destination(destination)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/workflows/builders/csv_parity_session_builder.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "csvtool/domain/csv_parity_session/source_pair"
+require "csvtool/domain/csv_parity_session/parity_options"
+require "csvtool/domain/csv_parity_session/parity_session"
+
+module Csvtool
+  module Interface
+    module CLI
+      module Workflows
+        module Builders
+          class CsvParitySessionBuilder
+            def call(left_path:, right_path:, col_sep:, headers_present:)
+              source_pair = Domain::CsvParitySession::SourcePair.new(
+                left_path: left_path,
+                right_path: right_path
+              )
+              options = Domain::CsvParitySession::ParityOptions.new(
+                separator: col_sep,
+                headers_present: headers_present
+              )
+
+              Domain::CsvParitySession::ParitySession.start(
+                source_pair: source_pair,
+                options: options
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/workflows/builders/row_extraction_session_builder.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "csvtool/domain/row_session/row_source"
+require "csvtool/domain/row_session/row_session"
+
+module Csvtool
+  module Interface
+    module CLI
+      module Workflows
+        module Builders
+          class RowExtractionSessionBuilder
+            def call(file_path:, col_sep:, row_range:, destination:)
+              source = Domain::RowSession::RowSource.new(path: file_path, separator: col_sep)
+              session = Domain::RowSession::RowSession.start(source: source, row_range: row_range)
+              session.with_output_destination(destination)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/workflows/builders/row_randomization_session_builder.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "csvtool/domain/row_randomization_session/randomization_source"
+require "csvtool/domain/row_randomization_session/randomization_options"
+require "csvtool/domain/row_randomization_session/randomization_session"
+
+module Csvtool
+  module Interface
+    module CLI
+      module Workflows
+        module Builders
+          class RowRandomizationSessionBuilder
+            def call(file_path:, col_sep:, headers_present:, seed:, destination:)
+              source = Domain::RowRandomizationSession::RandomizationSource.new(
+                path: file_path,
+                separator: col_sep,
+                headers_present: headers_present
+              )
+              options = Domain::RowRandomizationSession::RandomizationOptions.new(seed: seed)
+              session = Domain::RowRandomizationSession::RandomizationSession.start(source: source, options: options)
+              session.with_output_destination(destination)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/workflows/presenters/column_extraction_presenter.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Interface
+    module CLI
+      module Workflows
+        module Presenters
+          class ColumnExtractionPresenter
+            def initialize(stdout:)
+              @stdout = stdout
+            end
+
+            def print_value(value)
+              @stdout.puts value
+            end
+
+            def print_file_written(path)
+              @stdout.puts "Wrote output to #{path}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/workflows/presenters/cross_csv_dedupe_presenter.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Csvtool
+  module Interface
+    module CLI
+      module Workflows
+        module Presenters
+          class CrossCsvDedupePresenter
+            def initialize(stdout:, col_sep:)
+              @stdout = stdout
+              @col_sep = col_sep
+            end
+
+            def print_header(headers)
+              @stdout.puts
+              @stdout.puts ::CSV.generate_line(headers, row_sep: "", col_sep: @col_sep).chomp
+            end
+
+            def print_row(fields)
+              @stdout.puts ::CSV.generate_line(fields, row_sep: "", col_sep: @col_sep).chomp
+            end
+
+            def print_file_written(path)
+              @stdout.puts "Wrote output to #{path}"
+            end
+
+            def print_summary(stats)
+              @stdout.puts "Summary: source_rows=#{stats[:source_rows]} removed_rows=#{stats[:removed_rows]} kept_rows=#{stats[:kept_rows_count]}"
+              @stdout.puts "No rows removed; no matching keys found." if stats[:removed_rows].zero?
+              @stdout.puts "All source rows were removed by dedupe." if stats[:source_rows].positive? && stats[:kept_rows_count].zero?
+            end
+          end
+        end
+      end
+    end
+  end
+end