csvops 0.2.0.alpha → 0.4.0.alpha

data/lib/csvtool/application/use_cases/run_cross_csv_dedupe.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "csv"
+require "csvtool/infrastructure/csv/header_reader"
+require "csvtool/infrastructure/csv/cross_csv_deduper"
+require "csvtool/infrastructure/csv/selector_validator"
+
+module Csvtool
+  module Application
+    module UseCases
+      class RunCrossCsvDedupe
+        Result = Struct.new(:ok, :error, :data, keyword_init: true) do
+          def ok?
+            ok
+          end
+        end
+
+        def initialize(
+          header_reader: Infrastructure::CSV::HeaderReader.new,
+          deduper: Infrastructure::CSV::CrossCsvDeduper.new,
+          selector_validator: Infrastructure::CSV::SelectorValidator.new(header_reader: header_reader)
+        )
+          @header_reader = header_reader
+          @deduper = deduper
+          @selector_validator = selector_validator
+        end
+
+        def call(session:, on_header: nil, on_row: nil)
+          current_read_path = session.source.path
+          return failure(:column_not_found) unless @selector_validator.valid?(profile: session.source, selector: session.key_mapping.source_selector)
+
+          current_read_path = session.reference.path
+          return failure(:column_not_found) unless @selector_validator.valid?(profile: session.reference, selector: session.key_mapping.reference_selector)
+
+          source_headers = session.source.headers_present? ? @header_reader.call(file_path: session.source.path, col_sep: session.source.separator) : nil
+          current_read_path = session.source.path
+
+          if session.output_destination.file?
+            write_file(session: session, source_headers: source_headers)
+          else
+            on_header.call(source_headers) if on_header && source_headers
+            stats = @deduper.each_retained(**dedupe_options(session)) do |fields|
+              on_row.call(fields) if on_row
+            end
+            success(stats: stats)
+          end
+        rescue CSV::MalformedCSVError
+          failure(:could_not_parse_csv)
+        rescue Errno::EACCES
+          failure(:cannot_read_file, path: current_read_path || session.source.path)
+        end
+
+        private
+
+        def write_file(session:, source_headers:)
+          stats = nil
+          ::CSV.open(
+            session.output_destination.path,
+            "w",
+            write_headers: !source_headers.nil?,
+            headers: source_headers,
+            col_sep: session.source.separator
+          ) do |csv|
+            stats = @deduper.each_retained(**dedupe_options(session)) { |fields| csv << fields }
+          end
+          success(stats: stats, output_path: session.output_destination.path)
+        rescue Errno::EACCES, Errno::ENOENT => e
+          failure(:cannot_write_output_file, path: session.output_destination.path, error_class: e.class)
+        end
+
+        def dedupe_options(session)
+          {
+            source_path: session.source.path,
+            reference_path: session.reference.path,
+            source_selector: session.key_mapping.source_selector,
+            reference_selector: session.key_mapping.reference_selector,
+            source_col_sep: session.source.separator,
+            reference_col_sep: session.reference.separator,
+            match_options: session.match_options
+          }
+        end
+
+        def success(data)
+          Result.new(ok: true, error: nil, data: data)
+        end
+
+        def failure(code, data = {})
+          Result.new(ok: false, error: code, data: data)
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/application/use_cases/run_extraction.rb

@@ -19,8 +19,8 @@ require "csvtool/domain/column_session/column_selection"
 require "csvtool/domain/column_session/extraction_options"
 require "csvtool/domain/column_session/extraction_value"
 require "csvtool/domain/column_session/preview"
-require "csvtool/domain/column_session/output_destination"
 require "csvtool/domain/column_session/column_session"
+require "csvtool/domain/shared/output_destination"
 
 module Csvtool
   module Application
@@ -79,9 +79,9 @@ module Csvtool
           return if output_destination.nil?
           domain_destination =
             if output_destination[:mode] == :file
-              Domain::ColumnSession::OutputDestination.file(path: output_destination[:path])
+              Domain::Shared::OutputDestination.file(path: output_destination[:path])
             else
-              Domain::ColumnSession::OutputDestination.console
+              Domain::Shared::OutputDestination.console
             end
           session = session.with_output_destination(domain_destination)
 

data/lib/csvtool/application/use_cases/run_row_extraction.rb

@@ -11,8 +11,8 @@ require "csvtool/infrastructure/output/csv_row_console_writer"
 require "csvtool/infrastructure/output/csv_row_file_writer"
 require "csvtool/domain/row_session/row_range"
 require "csvtool/domain/row_session/row_source"
-require "csvtool/domain/row_session/row_output_destination"
 require "csvtool/domain/row_session/row_session"
+require "csvtool/domain/shared/output_destination"
 
 module Csvtool
   module Application
@@ -56,9 +56,9 @@ module Csvtool
           return if output_destination.nil?
           destination =
             if output_destination[:mode] == :file
-              Domain::RowSession::RowOutputDestination.file(path: output_destination[:path])
+              Domain::Shared::OutputDestination.file(path: output_destination[:path])
             else
-              Domain::RowSession::RowOutputDestination.console
+              Domain::Shared::OutputDestination.console
             end
           session = session.with_output_destination(destination)
 

data/lib/csvtool/application/use_cases/run_row_randomization.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require "csv"
+require "csvtool/interface/cli/errors/presenter"
+require "csvtool/interface/cli/prompts/file_path_prompt"
+require "csvtool/interface/cli/prompts/separator_prompt"
+require "csvtool/interface/cli/prompts/headers_present_prompt"
+require "csvtool/interface/cli/prompts/seed_prompt"
+require "csvtool/interface/cli/prompts/output_destination_prompt"
+require "csvtool/infrastructure/csv/header_reader"
+require "csvtool/infrastructure/csv/row_randomizer"
+require "csvtool/domain/row_randomization_session/randomization_source"
+require "csvtool/domain/row_randomization_session/randomization_options"
+require "csvtool/domain/row_randomization_session/randomization_session"
+require "csvtool/domain/shared/output_destination"
+
+module Csvtool
+  module Application
+    module UseCases
+      class RunRowRandomization
+        def initialize(stdin:, stdout:)
+          @stdin = stdin
+          @stdout = stdout
+          @errors = Interface::CLI::Errors::Presenter.new(stdout: stdout)
+          @header_reader = Infrastructure::CSV::HeaderReader.new
+          @row_randomizer = Infrastructure::CSV::RowRandomizer.new
+        end
+
+        def call
+          file_path = Interface::CLI::Prompts::FilePathPrompt.new(stdin: @stdin, stdout: @stdout).call
+          return @errors.file_not_found(file_path) unless File.file?(file_path)
+
+          col_sep = Interface::CLI::Prompts::SeparatorPrompt.new(stdin: @stdin, stdout: @stdout, errors: @errors).call
+          return if col_sep.nil?
+
+          headers_present = Interface::CLI::Prompts::HeadersPresentPrompt.new(stdin: @stdin, stdout: @stdout).call
+          source = Domain::RowRandomizationSession::RandomizationSource.new(
+            path: file_path,
+            separator: col_sep,
+            headers_present: headers_present
+          )
+          headers = source.headers_present? ? @header_reader.call(file_path: source.path, col_sep: source.separator) : nil
+          return @errors.no_headers if source.headers_present? && headers.empty?
+
+          seed = Interface::CLI::Prompts::SeedPrompt.new(stdin: @stdin, stdout: @stdout, errors: @errors).call
+          return if seed == Interface::CLI::Prompts::SeedPrompt::INVALID
+          options = Domain::RowRandomizationSession::RandomizationOptions.new(seed: seed)
+          session = Domain::RowRandomizationSession::RandomizationSession.start(source: source, options: options)
+
+          output_destination = Interface::CLI::Prompts::OutputDestinationPrompt.new(
+            stdin: @stdin,
+            stdout: @stdout,
+            errors: @errors
+          ).call
+          return if output_destination.nil?
+          destination =
+            if output_destination[:mode] == :file
+              Domain::Shared::OutputDestination.file(path: output_destination[:path])
+            else
+              Domain::Shared::OutputDestination.console
+            end
+          session = session.with_output_destination(destination)
+
+          randomized_rows = @row_randomizer.each(
+            file_path: session.source.path,
+            col_sep: session.source.separator,
+            headers: session.source.headers_present?,
+            seed: session.options.seed
+          )
+
+          if session.output_destination.file?
+            write_output_file(session.output_destination.path, headers, randomized_rows, col_sep: session.source.separator)
+          else
+            print_to_console(headers, randomized_rows, col_sep: session.source.separator)
+          end
+        rescue CSV::MalformedCSVError
+          @errors.could_not_parse_csv
+        rescue ArgumentError => e
+          return @errors.empty_output_path if e.message == "file output path cannot be empty"
+
+          raise e
+        rescue Errno::EACCES
+          @errors.cannot_read_file(file_path)
+        end
+
+        private
+
+        def print_to_console(headers, rows, col_sep:)
+          @stdout.puts
+          @stdout.puts ::CSV.generate_line(headers, row_sep: "", col_sep: col_sep).chomp if headers
+          rows.each { |fields| @stdout.puts ::CSV.generate_line(fields, row_sep: "", col_sep: col_sep).chomp }
+        end
+
+        def write_output_file(path, headers, rows, col_sep:)
+          ::CSV.open(path, "w", write_headers: !headers.nil?, headers: headers, col_sep: col_sep) do |csv|
+            rows.each { |fields| csv << fields }
+          end
+          @stdout.puts "Wrote output to #{path}"
+        rescue Errno::EACCES, Errno::ENOENT => e
+          @errors.cannot_write_output_file(path, e.class)
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/cli.rb

@@ -4,6 +4,8 @@ require "csv"
 require "csvtool/interface/cli/menu_loop"
 require "csvtool/application/use_cases/run_extraction"
 require "csvtool/application/use_cases/run_row_extraction"
+require "csvtool/application/use_cases/run_row_randomization"
+require "csvtool/interface/cli/workflows/run_cross_csv_dedupe_workflow"
 require "csvtool/interface/cli/errors/presenter"
 require "csvtool/infrastructure/csv/header_reader"
 require "csvtool/infrastructure/csv/value_streamer"
@@ -14,6 +16,8 @@ module Csvtool
     MENU_OPTIONS = [
       "Extract column",
       "Extract rows (range)",
+      "Randomize rows",
+      "Dedupe using another CSV",
       "Exit"
     ].freeze
 
@@ -45,12 +49,16 @@ module Csvtool
     def run_menu_loop
       extract_column_action = -> { Application::UseCases::RunExtraction.new(stdin: @stdin, stdout: @stdout).call }
       extract_rows_action = -> { Application::UseCases::RunRowExtraction.new(stdin: @stdin, stdout: @stdout).call }
+      randomize_rows_action = -> { Application::UseCases::RunRowRandomization.new(stdin: @stdin, stdout: @stdout).call }
+      dedupe_action = -> { Interface::CLI::Workflows::RunCrossCsvDedupeWorkflow.new(stdin: @stdin, stdout: @stdout).call }
       Interface::CLI::MenuLoop.new(
         stdin: @stdin,
         stdout: @stdout,
         menu_options: MENU_OPTIONS,
         extract_column_action: extract_column_action,
-        extract_rows_action: extract_rows_action
+        extract_rows_action: extract_rows_action,
+        randomize_rows_action: randomize_rows_action,
+        dedupe_action: dedupe_action
       ).run
     end
 

data/lib/csvtool/domain/cross_csv_dedupe_session/column_selector.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Domain
+    module CrossCsvDedupeSession
+      class ColumnSelector
+        attr_reader :value
+
+        def self.from_input(headers_present:, input:)
+          if headers_present
+            raise ArgumentError, "column name cannot be empty" if input.to_s.empty?
+
+            new(value: input.to_s, headers_present: true)
+          else
+            raise ArgumentError, "column index must be a positive integer" unless /\A[1-9]\d*\z/.match?(input.to_s)
+
+            new(value: input.to_i, headers_present: false)
+          end
+        end
+
+        def initialize(value:, headers_present:)
+          @value = value
+          @headers_present = !!headers_present
+        end
+
+        def headers_present?
+          @headers_present
+        end
+
+        def index?
+          !@headers_present
+        end
+
+        def extract_from(row)
+          if headers_present?
+            row[@value].to_s
+          else
+            row[@value - 1].to_s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/domain/cross_csv_dedupe_session/cross_csv_dedupe_session.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "csvtool/domain/cross_csv_dedupe_session/csv_profile"
+require "csvtool/domain/cross_csv_dedupe_session/key_mapping"
+require "csvtool/domain/cross_csv_dedupe_session/match_options"
+require "csvtool/domain/shared/output_destination"
+
+module Csvtool
+  module Domain
+    module CrossCsvDedupeSession
+      class CrossCsvDedupeSession
+        attr_reader :source, :reference, :key_mapping, :match_options, :output_destination
+
+        def self.start(source:, reference:, key_mapping:, match_options:)
+          new(source: source, reference: reference, key_mapping: key_mapping, match_options: match_options)
+        end
+
+        def initialize(source:, reference:, key_mapping:, match_options:, output_destination: nil)
+          raise ArgumentError, "source must be CsvProfile" unless source.is_a?(CsvProfile)
+          raise ArgumentError, "reference must be CsvProfile" unless reference.is_a?(CsvProfile)
+          raise ArgumentError, "key_mapping must be KeyMapping" unless key_mapping.is_a?(KeyMapping)
+          raise ArgumentError, "match_options must be MatchOptions" unless match_options.is_a?(MatchOptions)
+          unless output_destination.nil? || output_destination.is_a?(Domain::Shared::OutputDestination)
+            raise ArgumentError, "output_destination must be OutputDestination or nil"
+          end
+
+          @source = source
+          @reference = reference
+          @key_mapping = key_mapping
+          @match_options = match_options
+          @output_destination = output_destination
+        end
+
+        def with_output_destination(destination)
+          self.class.new(
+            source: @source,
+            reference: @reference,
+            key_mapping: @key_mapping,
+            match_options: @match_options,
+            output_destination: destination
+          )
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/domain/cross_csv_dedupe_session/csv_profile.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Domain
+    module CrossCsvDedupeSession
+      class CsvProfile
+        attr_reader :path, :separator
+
+        def initialize(path:, separator:, headers_present:)
+          raise ArgumentError, "path cannot be empty" if path.to_s.empty?
+          raise ArgumentError, "separator cannot be empty" if separator.to_s.empty?
+
+          @path = path
+          @separator = separator
+          @headers_present = !!headers_present
+        end
+
+        def headers_present?
+          @headers_present
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/domain/cross_csv_dedupe_session/key_mapping.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "csvtool/domain/cross_csv_dedupe_session/column_selector"
+
+module Csvtool
+  module Domain
+    module CrossCsvDedupeSession
+      class KeyMapping
+        attr_reader :source_selector, :reference_selector
+
+        def initialize(source_selector:, reference_selector:)
+          unless source_selector.is_a?(ColumnSelector) && reference_selector.is_a?(ColumnSelector)
+            raise ArgumentError, "selectors must be ColumnSelector"
+          end
+
+          @source_selector = source_selector
+          @reference_selector = reference_selector
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/domain/cross_csv_dedupe_session/match_options.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Domain
+    module CrossCsvDedupeSession
+      class MatchOptions
+        attr_reader :trim_whitespace, :case_insensitive
+
+        def initialize(trim_whitespace:, case_insensitive:)
+          @trim_whitespace = !!trim_whitespace
+          @case_insensitive = !!case_insensitive
+        end
+
+        def trim_whitespace?
+          @trim_whitespace
+        end
+
+        def case_insensitive?
+          @case_insensitive
+        end
+
+        def normalize(value)
+          normalized = trim_whitespace? ? value.to_s.strip : value.to_s
+          case_insensitive? ? normalized.downcase : normalized
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/domain/row_randomization_session/randomization_options.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Domain
+    module RowRandomizationSession
+      class RandomizationOptions
+        attr_reader :seed
+
+        def initialize(seed:)
+          raise ArgumentError, "seed must be an integer or nil" unless seed.nil? || seed.is_a?(Integer)
+
+          @seed = seed
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/domain/row_randomization_session/randomization_session.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Domain
+    module RowRandomizationSession
+      class RandomizationSession
+        attr_reader :source, :options, :output_destination
+
+        def self.start(source:, options:)
+          new(source: source, options: options)
+        end
+
+        def initialize(source:, options:, output_destination: nil)
+          @source = source
+          @options = options
+          @output_destination = output_destination
+        end
+
+        def with_output_destination(destination)
+          self.class.new(source: @source, options: @options, output_destination: destination)
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/domain/row_randomization_session/randomization_source.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Domain
+    module RowRandomizationSession
+      class RandomizationSource
+        attr_reader :path, :separator
+
+        def initialize(path:, separator:, headers_present:)
+          raise ArgumentError, "path cannot be empty" if path.to_s.empty?
+          raise ArgumentError, "separator cannot be empty" if separator.to_s.empty?
+
+          @path = path
+          @separator = separator
+          @headers_present = headers_present
+        end
+
+        def headers_present?
+          @headers_present
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/domain/row_session/row_source.rb

@@ -7,6 +7,9 @@ module Csvtool
         attr_reader :path, :separator
 
         def initialize(path:, separator:)
+          raise ArgumentError, "path cannot be empty" if path.to_s.empty?
+          raise ArgumentError, "separator cannot be empty" if separator.to_s.empty?
+
           @path = path
           @separator = separator
         end

data/lib/csvtool/domain/{column_session → shared}/output_destination.rb

@@ -2,7 +2,7 @@
 
 module Csvtool
   module Domain
-    module ColumnSession
+    module Shared
       class OutputDestination
         attr_reader :mode, :path
 

data/lib/csvtool/infrastructure/csv/cross_csv_deduper.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "csv"
+require "set"
+require "csvtool/domain/cross_csv_dedupe_session/match_options"
+
+module Csvtool
+  module Infrastructure
+    module CSV
+      class CrossCsvDeduper
+        def call(
+          source_path:,
+          reference_path:,
+          source_selector:,
+          reference_selector:,
+          source_col_sep: ",",
+          reference_col_sep: ",",
+          match_options: Domain::CrossCsvDedupeSession::MatchOptions.new(trim_whitespace: true, case_insensitive: false)
+        )
+          kept_rows = []
+          stats = each_retained(
+            source_path: source_path,
+            reference_path: reference_path,
+            source_selector: source_selector,
+            reference_selector: reference_selector,
+            source_col_sep: source_col_sep,
+            reference_col_sep: reference_col_sep,
+            match_options: match_options
+          ) do |fields|
+            kept_rows << fields
+          end
+
+          stats.merge(kept_rows: kept_rows)
+        end
+
+        def each_retained(
+          source_path:,
+          reference_path:,
+          source_selector:,
+          reference_selector:,
+          source_col_sep: ",",
+          reference_col_sep: ",",
+          match_options: Domain::CrossCsvDedupeSession::MatchOptions.new(trim_whitespace: true, case_insensitive: false)
+        )
+          source_has_headers = source_selector.headers_present?
+          reference_has_headers = reference_selector.headers_present?
+          reference_keys = Set.new
+          ::CSV.foreach(reference_path, headers: reference_has_headers, col_sep: reference_col_sep) do |row|
+            reference_keys << extract_key(row, selector: reference_selector, match_options: match_options)
+          end
+
+          source_header_row = nil
+          source_rows = 0
+          removed_rows = 0
+          kept_rows_count = 0
+
+          ::CSV.foreach(source_path, headers: source_has_headers, col_sep: source_col_sep) do |row|
+            source_header_row ||= row.headers if source_has_headers
+            source_rows += 1
+            key = extract_key(row, selector: source_selector, match_options: match_options)
+            if reference_keys.include?(key)
+              removed_rows += 1
+            else
+              kept_rows_count += 1
+              yield(source_has_headers ? row.fields : row) if block_given?
+            end
+          end
+
+          {
+            headers: source_has_headers ? (source_header_row || []) : nil,
+            source_rows: source_rows,
+            removed_rows: removed_rows,
+            kept_rows_count: kept_rows_count
+          }
+        end
+
+        private
+
+        def extract_key(row, selector:, match_options:)
+          match_options.normalize(selector.extract_from(row))
+        end
+      end
+    end
+  end
+end