csvops 0.3.0.alpha → 0.5.0.alpha

data/lib/csvtool/application/use_cases/run_row_randomization.rb

@@ -1,103 +1,86 @@
 # frozen_string_literal: true
 
 require "csv"
-require "csvtool/interface/cli/errors/presenter"
-require "csvtool/interface/cli/prompts/file_path_prompt"
-require "csvtool/interface/cli/prompts/separator_prompt"
-require "csvtool/interface/cli/prompts/headers_present_prompt"
-require "csvtool/interface/cli/prompts/seed_prompt"
-require "csvtool/interface/cli/prompts/output_destination_prompt"
 require "csvtool/infrastructure/csv/header_reader"
 require "csvtool/infrastructure/csv/row_randomizer"
-require "csvtool/domain/row_randomization_session/randomization_source"
-require "csvtool/domain/row_randomization_session/randomization_options"
-require "csvtool/domain/row_randomization_session/randomization_output_destination"
-require "csvtool/domain/row_randomization_session/randomization_session"
+require "csvtool/infrastructure/output/csv_randomized_row_file_writer"
 
 module Csvtool
   module Application
     module UseCases
       class RunRowRandomization
-        def initialize(stdin:, stdout:)
-          @stdin = stdin
-          @stdout = stdout
-          @errors = Interface::CLI::Errors::Presenter.new(stdout: stdout)
-          @header_reader = Infrastructure::CSV::HeaderReader.new
-          @row_randomizer = Infrastructure::CSV::RowRandomizer.new
+        Result = Struct.new(:ok, :error, :data, keyword_init: true) do
+          def ok?
+            ok
+          end
         end
 
-        def call
-          file_path = Interface::CLI::Prompts::FilePathPrompt.new(stdin: @stdin, stdout: @stdout).call
-          return @errors.file_not_found(file_path) unless File.file?(file_path)
-
-          col_sep = Interface::CLI::Prompts::SeparatorPrompt.new(stdin: @stdin, stdout: @stdout, errors: @errors).call
-          return if col_sep.nil?
-
-          headers_present = Interface::CLI::Prompts::HeadersPresentPrompt.new(stdin: @stdin, stdout: @stdout).call
-          source = Domain::RowRandomizationSession::RandomizationSource.new(
-            path: file_path,
-            separator: col_sep,
-            headers_present: headers_present
+        def initialize(
+          header_reader: Infrastructure::CSV::HeaderReader.new,
+          row_randomizer: Infrastructure::CSV::RowRandomizer.new,
+          csv_randomized_row_file_writer: nil
+        )
+          @header_reader = header_reader
+          @row_randomizer = row_randomizer
+          @csv_randomized_row_file_writer = csv_randomized_row_file_writer || Infrastructure::Output::CsvRandomizedRowFileWriter.new(
+            row_randomizer: @row_randomizer
           )
-          headers = source.headers_present? ? @header_reader.call(file_path: source.path, col_sep: source.separator) : nil
-          return @errors.no_headers if source.headers_present? && headers.empty?
+        end
 
-          seed = Interface::CLI::Prompts::SeedPrompt.new(stdin: @stdin, stdout: @stdout, errors: @errors).call
-          return if seed == Interface::CLI::Prompts::SeedPrompt::INVALID
-          options = Domain::RowRandomizationSession::RandomizationOptions.new(seed: seed)
-          session = Domain::RowRandomizationSession::RandomizationSession.start(source: source, options: options)
+        def read_headers(file_path:, col_sep:, headers_present:)
+          return failure(:file_not_found, path: file_path) unless File.file?(file_path)
 
-          output_destination = Interface::CLI::Prompts::OutputDestinationPrompt.new(
-            stdin: @stdin,
-            stdout: @stdout,
-            errors: @errors
-          ).call
-          return if output_destination.nil?
-          destination =
-            if output_destination[:mode] == :file
-              Domain::RowRandomizationSession::RandomizationOutputDestination.file(path: output_destination[:path])
-            else
-              Domain::RowRandomizationSession::RandomizationOutputDestination.console
-            end
-          session = session.with_output_destination(destination)
+          headers = nil
+          if headers_present
+            headers = @header_reader.call(file_path: file_path, col_sep: col_sep)
+            return failure(:no_headers) if headers.empty?
+          end
 
-          randomized_rows = @row_randomizer.each(
-            file_path: session.source.path,
-            col_sep: session.source.separator,
-            headers: session.source.headers_present?,
-            seed: session.options.seed
-          )
+          success(headers: headers)
+        rescue CSV::MalformedCSVError
+          failure(:could_not_parse_csv)
+        rescue Errno::EACCES
+          failure(:cannot_read_file, path: file_path)
+        end
 
+        def randomize(session:, headers:, on_row: nil)
           if session.output_destination.file?
-            write_output_file(session.output_destination.path, headers, randomized_rows, col_sep: session.source.separator)
+            @csv_randomized_row_file_writer.call(
+              path: session.output_destination.path,
+              headers: headers,
+              file_path: session.source.path,
+              col_sep: session.source.separator,
+              headers_present: session.source.headers_present?,
+              seed: session.options.seed
+            )
+            success(output_path: session.output_destination.path)
           else
-            print_to_console(headers, randomized_rows, col_sep: session.source.separator)
+            @row_randomizer.each(
+              file_path: session.source.path,
+              col_sep: session.source.separator,
+              headers: session.source.headers_present?,
+              seed: session.options.seed
+            ) { |fields| on_row.call(fields) if on_row }
+            success({})
           end
         rescue CSV::MalformedCSVError
-          @errors.could_not_parse_csv
-        rescue ArgumentError => e
-          return @errors.empty_output_path if e.message == "file output path cannot be empty"
-
-          raise e
-        rescue Errno::EACCES
-          @errors.cannot_read_file(file_path)
+          failure(:could_not_parse_csv)
+        rescue Errno::EACCES, Errno::ENOENT => e
+          if session.output_destination.file?
+            failure(:cannot_write_output_file, path: session.output_destination.path, error_class: e.class)
+          else
+            failure(:cannot_read_file, path: session.source.path)
+          end
         end
 
         private
 
-        def print_to_console(headers, rows, col_sep:)
-          @stdout.puts
-          @stdout.puts ::CSV.generate_line(headers, row_sep: "", col_sep: col_sep).chomp if headers
-          rows.each { |fields| @stdout.puts ::CSV.generate_line(fields, row_sep: "", col_sep: col_sep).chomp }
+        def success(data)
+          Result.new(ok: true, error: nil, data: data)
         end
 
-        def write_output_file(path, headers, rows, col_sep:)
-          ::CSV.open(path, "w", write_headers: !headers.nil?, headers: headers, col_sep: col_sep) do |csv|
-            rows.each { |fields| csv << fields }
-          end
-          @stdout.puts "Wrote output to #{path}"
-        rescue Errno::EACCES, Errno::ENOENT => e
-          @errors.cannot_write_output_file(path, e.class)
+        def failure(code, data = {})
+          Result.new(ok: false, error: code, data: data)
         end
       end
     end

data/lib/csvtool/cli.rb

@@ -2,9 +2,10 @@
 
 require "csv"
 require "csvtool/interface/cli/menu_loop"
-require "csvtool/application/use_cases/run_extraction"
-require "csvtool/application/use_cases/run_row_extraction"
-require "csvtool/application/use_cases/run_row_randomization"
+require "csvtool/interface/cli/workflows/run_extraction_workflow"
+require "csvtool/interface/cli/workflows/run_row_extraction_workflow"
+require "csvtool/interface/cli/workflows/run_row_randomization_workflow"
+require "csvtool/interface/cli/workflows/run_cross_csv_dedupe_workflow"
 require "csvtool/interface/cli/errors/presenter"
 require "csvtool/infrastructure/csv/header_reader"
 require "csvtool/infrastructure/csv/value_streamer"
@@ -16,6 +17,7 @@ module Csvtool
       "Extract column",
       "Extract rows (range)",
       "Randomize rows",
+      "Dedupe using another CSV",
       "Exit"
     ].freeze
 
@@ -45,16 +47,18 @@ module Csvtool
     private
 
     def run_menu_loop
-      extract_column_action = -> { Application::UseCases::RunExtraction.new(stdin: @stdin, stdout: @stdout).call }
-      extract_rows_action = -> { Application::UseCases::RunRowExtraction.new(stdin: @stdin, stdout: @stdout).call }
-      randomize_rows_action = -> { Application::UseCases::RunRowRandomization.new(stdin: @stdin, stdout: @stdout).call }
+      extract_column_action = -> { Interface::CLI::Workflows::RunExtractionWorkflow.new(stdin: @stdin, stdout: @stdout).call }
+      extract_rows_action = -> { Interface::CLI::Workflows::RunRowExtractionWorkflow.new(stdin: @stdin, stdout: @stdout).call }
+      randomize_rows_action = -> { Interface::CLI::Workflows::RunRowRandomizationWorkflow.new(stdin: @stdin, stdout: @stdout).call }
+      dedupe_action = -> { Interface::CLI::Workflows::RunCrossCsvDedupeWorkflow.new(stdin: @stdin, stdout: @stdout).call }
       Interface::CLI::MenuLoop.new(
         stdin: @stdin,
         stdout: @stdout,
         menu_options: MENU_OPTIONS,
         extract_column_action: extract_column_action,
         extract_rows_action: extract_rows_action,
-        randomize_rows_action: randomize_rows_action
+        randomize_rows_action: randomize_rows_action,
+        dedupe_action: dedupe_action
       ).run
     end
 

data/lib/csvtool/domain/cross_csv_dedupe_session/column_selector.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Domain
+    module CrossCsvDedupeSession
+      class ColumnSelector
+        attr_reader :value
+
+        def self.from_input(headers_present:, input:)
+          if headers_present
+            raise ArgumentError, "column name cannot be empty" if input.to_s.empty?
+
+            new(value: input.to_s, headers_present: true)
+          else
+            raise ArgumentError, "column index must be a positive integer" unless /\A[1-9]\d*\z/.match?(input.to_s)
+
+            new(value: input.to_i, headers_present: false)
+          end
+        end
+
+        def initialize(value:, headers_present:)
+          @value = value
+          @headers_present = !!headers_present
+        end
+
+        def headers_present?
+          @headers_present
+        end
+
+        def index?
+          !@headers_present
+        end
+
+        def extract_from(row)
+          if headers_present?
+            row[@value].to_s
+          else
+            row[@value - 1].to_s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/domain/cross_csv_dedupe_session/cross_csv_dedupe_session.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "csvtool/domain/cross_csv_dedupe_session/csv_profile"
+require "csvtool/domain/cross_csv_dedupe_session/key_mapping"
+require "csvtool/domain/cross_csv_dedupe_session/match_options"
+require "csvtool/domain/shared/output_destination"
+
+module Csvtool
+  module Domain
+    module CrossCsvDedupeSession
+      class CrossCsvDedupeSession
+        attr_reader :source, :reference, :key_mapping, :match_options, :output_destination
+
+        def self.start(source:, reference:, key_mapping:, match_options:)
+          new(source: source, reference: reference, key_mapping: key_mapping, match_options: match_options)
+        end
+
+        def initialize(source:, reference:, key_mapping:, match_options:, output_destination: nil)
+          raise ArgumentError, "source must be CsvProfile" unless source.is_a?(CsvProfile)
+          raise ArgumentError, "reference must be CsvProfile" unless reference.is_a?(CsvProfile)
+          raise ArgumentError, "key_mapping must be KeyMapping" unless key_mapping.is_a?(KeyMapping)
+          raise ArgumentError, "match_options must be MatchOptions" unless match_options.is_a?(MatchOptions)
+          unless output_destination.nil? || output_destination.is_a?(Domain::Shared::OutputDestination)
+            raise ArgumentError, "output_destination must be OutputDestination or nil"
+          end
+
+          @source = source
+          @reference = reference
+          @key_mapping = key_mapping
+          @match_options = match_options
+          @output_destination = output_destination
+        end
+
+        def with_output_destination(destination)
+          self.class.new(
+            source: @source,
+            reference: @reference,
+            key_mapping: @key_mapping,
+            match_options: @match_options,
+            output_destination: destination
+          )
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/domain/cross_csv_dedupe_session/csv_profile.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Domain
+    module CrossCsvDedupeSession
+      class CsvProfile
+        attr_reader :path, :separator
+
+        def initialize(path:, separator:, headers_present:)
+          raise ArgumentError, "path cannot be empty" if path.to_s.empty?
+          raise ArgumentError, "separator cannot be empty" if separator.to_s.empty?
+
+          @path = path
+          @separator = separator
+          @headers_present = !!headers_present
+        end
+
+        def headers_present?
+          @headers_present
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/domain/cross_csv_dedupe_session/key_mapping.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "csvtool/domain/cross_csv_dedupe_session/column_selector"
+
+module Csvtool
+  module Domain
+    module CrossCsvDedupeSession
+      class KeyMapping
+        attr_reader :source_selector, :reference_selector
+
+        def initialize(source_selector:, reference_selector:)
+          unless source_selector.is_a?(ColumnSelector) && reference_selector.is_a?(ColumnSelector)
+            raise ArgumentError, "selectors must be ColumnSelector"
+          end
+
+          @source_selector = source_selector
+          @reference_selector = reference_selector
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/domain/cross_csv_dedupe_session/match_options.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Domain
+    module CrossCsvDedupeSession
+      class MatchOptions
+        attr_reader :trim_whitespace, :case_insensitive
+
+        def initialize(trim_whitespace:, case_insensitive:)
+          @trim_whitespace = !!trim_whitespace
+          @case_insensitive = !!case_insensitive
+        end
+
+        def trim_whitespace?
+          @trim_whitespace
+        end
+
+        def case_insensitive?
+          @case_insensitive
+        end
+
+        def normalize(value)
+          normalized = trim_whitespace? ? value.to_s.strip : value.to_s
+          case_insensitive? ? normalized.downcase : normalized
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/domain/row_randomization_session/randomization_source.rb

@@ -7,6 +7,7 @@ module Csvtool
         attr_reader :path, :separator
 
         def initialize(path:, separator:, headers_present:)
+          raise ArgumentError, "path cannot be empty" if path.to_s.empty?
           raise ArgumentError, "separator cannot be empty" if separator.to_s.empty?
 
           @path = path

data/lib/csvtool/domain/row_session/row_source.rb

@@ -7,6 +7,9 @@ module Csvtool
         attr_reader :path, :separator
 
         def initialize(path:, separator:)
+          raise ArgumentError, "path cannot be empty" if path.to_s.empty?
+          raise ArgumentError, "separator cannot be empty" if separator.to_s.empty?
+
           @path = path
           @separator = separator
         end

data/lib/csvtool/domain/{column_session → shared}/output_destination.rb

@@ -2,7 +2,7 @@
 
 module Csvtool
   module Domain
-    module ColumnSession
+    module Shared
       class OutputDestination
         attr_reader :mode, :path
 

data/lib/csvtool/infrastructure/csv/cross_csv_deduper.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "csv"
+require "set"
+require "csvtool/domain/cross_csv_dedupe_session/match_options"
+
+module Csvtool
+  module Infrastructure
+    module CSV
+      class CrossCsvDeduper
+        def call(
+          source_path:,
+          reference_path:,
+          source_selector:,
+          reference_selector:,
+          source_col_sep: ",",
+          reference_col_sep: ",",
+          match_options: Domain::CrossCsvDedupeSession::MatchOptions.new(trim_whitespace: true, case_insensitive: false)
+        )
+          kept_rows = []
+          stats = each_retained(
+            source_path: source_path,
+            reference_path: reference_path,
+            source_selector: source_selector,
+            reference_selector: reference_selector,
+            source_col_sep: source_col_sep,
+            reference_col_sep: reference_col_sep,
+            match_options: match_options
+          ) do |fields|
+            kept_rows << fields
+          end
+
+          stats.merge(kept_rows: kept_rows)
+        end
+
+        def each_retained(
+          source_path:,
+          reference_path:,
+          source_selector:,
+          reference_selector:,
+          source_col_sep: ",",
+          reference_col_sep: ",",
+          match_options: Domain::CrossCsvDedupeSession::MatchOptions.new(trim_whitespace: true, case_insensitive: false)
+        )
+          source_has_headers = source_selector.headers_present?
+          reference_has_headers = reference_selector.headers_present?
+          reference_keys = Set.new
+          ::CSV.foreach(reference_path, headers: reference_has_headers, col_sep: reference_col_sep) do |row|
+            reference_keys << extract_key(row, selector: reference_selector, match_options: match_options)
+          end
+
+          source_header_row = nil
+          source_rows = 0
+          removed_rows = 0
+          kept_rows_count = 0
+
+          ::CSV.foreach(source_path, headers: source_has_headers, col_sep: source_col_sep) do |row|
+            source_header_row ||= row.headers if source_has_headers
+            source_rows += 1
+            key = extract_key(row, selector: source_selector, match_options: match_options)
+            if reference_keys.include?(key)
+              removed_rows += 1
+            else
+              kept_rows_count += 1
+              yield(source_has_headers ? row.fields : row) if block_given?
+            end
+          end
+
+          {
+            headers: source_has_headers ? (source_header_row || []) : nil,
+            source_rows: source_rows,
+            removed_rows: removed_rows,
+            kept_rows_count: kept_rows_count
+          }
+        end
+
+        private
+
+        def extract_key(row, selector:, match_options:)
+          match_options.normalize(selector.extract_from(row))
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/infrastructure/csv/selector_validator.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "csv"
+require "csvtool/infrastructure/csv/header_reader"
+
+module Csvtool
+  module Infrastructure
+    module CSV
+      class SelectorValidator
+        def initialize(header_reader: HeaderReader.new)
+          @header_reader = header_reader
+        end
+
+        def valid?(profile:, selector:)
+          if selector.headers_present?
+            headers = @header_reader.call(file_path: profile.path, col_sep: profile.separator)
+            return false if headers.empty?
+
+            headers.include?(selector.value)
+          else
+            first_row = ::CSV.open(profile.path, "r", headers: false, col_sep: profile.separator, &:first)
+            return false if first_row.nil?
+
+            selector.value <= first_row.length
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/infrastructure/output/csv_cross_csv_dedupe_file_writer.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Csvtool
+  module Infrastructure
+    module Output
+      class CsvCrossCsvDedupeFileWriter
+        def initialize(deduper:)
+          @deduper = deduper
+        end
+
+        def call(path:, headers:, col_sep:, dedupe_options:)
+          stats = nil
+          ::CSV.open(path, "w", write_headers: !headers.nil?, headers: headers, col_sep: col_sep) do |csv|
+            stats = @deduper.each_retained(**dedupe_options) { |fields| csv << fields }
+          end
+          stats
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/infrastructure/output/csv_file_writer.rb

@@ -6,9 +6,7 @@ module Csvtool
   module Infrastructure
     module Output
       class CsvFileWriter
-        def initialize(stdout:, errors:, value_streamer:)
-          @stdout = stdout
-          @errors = errors
+        def initialize(value_streamer:)
           @value_streamer = value_streamer
         end
 
@@ -19,10 +17,6 @@ module Csvtool
               csv << [value]
             end
           end
-
-          @stdout.puts "Wrote output to #{output_path}"
-        rescue Errno::EACCES, Errno::ENOENT => e
-          @errors.cannot_write_output_file(output_path, e.class)
         end
       end
     end

data/lib/csvtool/infrastructure/output/csv_randomized_row_file_writer.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Csvtool
+  module Infrastructure
+    module Output
+      class CsvRandomizedRowFileWriter
+        def initialize(row_randomizer:)
+          @row_randomizer = row_randomizer
+        end
+
+        def call(path:, headers:, file_path:, col_sep:, headers_present:, seed:)
+          ::CSV.open(path, "w", write_headers: !headers.nil?, headers: headers, col_sep: col_sep) do |csv|
+            @row_randomizer.each(file_path: file_path, col_sep: col_sep, headers: headers_present, seed: seed) do |fields|
+              csv << fields
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/infrastructure/output/csv_row_file_writer.rb

@@ -6,9 +6,7 @@ module Csvtool
   module Infrastructure
     module Output
       class CsvRowFileWriter
-        def initialize(stdout:, errors:, row_streamer:)
-          @stdout = stdout
-          @errors = errors
+        def initialize(row_streamer:)
           @row_streamer = row_streamer
         end
 
@@ -30,12 +28,7 @@ module Csvtool
             csv << fields
           end
 
-          csv&.close
-          @stdout.puts "Wrote output to #{output_path}" if wrote_rows
-          stats
-        rescue Errno::EACCES, Errno::ENOENT => e
-          @errors.cannot_write_output_file(output_path, e.class)
-          nil
+          stats.merge(wrote_rows: wrote_rows)
         ensure
           csv&.close unless csv&.closed?
         end

data/lib/csvtool/interface/cli/menu_loop.rb

@@ -4,13 +4,14 @@ module Csvtool
   module Interface
     module CLI
       class MenuLoop
-        def initialize(stdin:, stdout:, menu_options:, extract_column_action:, extract_rows_action:, randomize_rows_action:)
+        def initialize(stdin:, stdout:, menu_options:, extract_column_action:, extract_rows_action:, randomize_rows_action:, dedupe_action:)
           @stdin = stdin
           @stdout = stdout
           @menu_options = menu_options
           @extract_column_action = extract_column_action
           @extract_rows_action = extract_rows_action
           @randomize_rows_action = randomize_rows_action
+          @dedupe_action = dedupe_action
         end
 
         def run
@@ -28,9 +29,11 @@ module Csvtool
             when "3"
               @randomize_rows_action.call
             when "4"
+              @dedupe_action.call
+            when "5"
               return 0
             else
-              @stdout.puts "Please choose 1, 2, 3, or 4."
+              @stdout.puts "Please choose 1, 2, 3, 4, or 5."
             end
           end
         end