csvops 0.2.0.alpha → 0.4.0.alpha

data/lib/csvtool/infrastructure/csv/row_randomizer.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require "csv"
+require "tempfile"
+
+module Csvtool
+  module Infrastructure
+    module CSV
+      class RowRandomizer
+        DEFAULT_CHUNK_SIZE = 10_000
+
+        def call(file_path:, col_sep:, headers:, seed: nil)
+          each(file_path: file_path, col_sep: col_sep, headers: headers, seed: seed).to_a
+        end
+
+        def each(file_path:, col_sep:, headers:, seed: nil, chunk_size: DEFAULT_CHUNK_SIZE)
+          chunk_paths = []
+          return enum_for(:each, file_path: file_path, col_sep: col_sep, headers: headers, seed: seed, chunk_size: chunk_size) unless block_given?
+
+          rng = seed.nil? ? Random.new : Random.new(seed)
+          sequence = 0
+          chunk_entries = []
+
+          ::CSV.foreach(file_path, headers: headers, col_sep: col_sep) do |row|
+            fields = headers ? row.fields : row
+            chunk_entries << [rng.rand, sequence, fields]
+            sequence += 1
+            flush_chunk(chunk_entries, chunk_paths) if chunk_entries.length >= chunk_size
+          end
+
+          flush_chunk(chunk_entries, chunk_paths) unless chunk_entries.empty?
+          merge_chunks(chunk_paths) { |fields| yield fields }
+        ensure
+          cleanup_chunks(chunk_paths)
+        end
+
+        private
+
+        def flush_chunk(entries, chunk_paths)
+          entries.sort_by! { |rand_key, seq, _fields| [rand_key, seq] }
+          file = Tempfile.new("csvtool-row-randomizer-chunk")
+          file.binmode
+          entries.each { |entry| Marshal.dump(entry, file) }
+          file.close
+          chunk_paths << file.path
+          entries.clear
+        end
+
+        def merge_chunks(chunk_paths)
+          readers = chunk_paths.map { |path| File.open(path, "rb") }
+          heads = readers.map { |reader| next_entry(reader) }
+
+          loop do
+            indexed = heads.each_with_index.select { |entry, _i| !entry.nil? }
+            break if indexed.empty?
+
+            min_entry, min_index = indexed.min_by { |entry, _i| [entry[0], entry[1]] }
+            yield min_entry[2]
+            heads[min_index] = next_entry(readers[min_index])
+          end
+        ensure
+          readers&.each(&:close)
+        end
+
+        def next_entry(reader)
+          Marshal.load(reader)
+        rescue EOFError
+          nil
+        end
+
+        def cleanup_chunks(chunk_paths)
+          return if chunk_paths.nil?
+
+          chunk_paths.each do |path|
+            File.delete(path) if File.exist?(path)
+          rescue Errno::EACCES, Errno::ENOENT
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/infrastructure/csv/selector_validator.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "csv"
+require "csvtool/infrastructure/csv/header_reader"
+
+module Csvtool
+  module Infrastructure
+    module CSV
+      class SelectorValidator
+        def initialize(header_reader: HeaderReader.new)
+          @header_reader = header_reader
+        end
+
+        def valid?(profile:, selector:)
+          if selector.headers_present?
+            headers = @header_reader.call(file_path: profile.path, col_sep: profile.separator)
+            return false if headers.empty?
+
+            headers.include?(selector.value)
+          else
+            first_row = ::CSV.open(profile.path, "r", headers: false, col_sep: profile.separator, &:first)
+            return false if first_row.nil?
+
+            selector.value <= first_row.length
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/errors/presenter.rb

@@ -49,6 +49,10 @@ module Csvtool
             @stdout.puts "Invalid separator choice."
           end
 
+          def invalid_seed
+            @stdout.puts "Seed must be an integer."
+          end
+
           def canceled
             @stdout.puts "Canceled."
           end

data/lib/csvtool/interface/cli/menu_loop.rb

@@ -4,12 +4,14 @@ module Csvtool
   module Interface
     module CLI
       class MenuLoop
-        def initialize(stdin:, stdout:, menu_options:, extract_column_action:, extract_rows_action:)
+        def initialize(stdin:, stdout:, menu_options:, extract_column_action:, extract_rows_action:, randomize_rows_action:, dedupe_action:)
           @stdin = stdin
           @stdout = stdout
           @menu_options = menu_options
           @extract_column_action = extract_column_action
           @extract_rows_action = extract_rows_action
+          @randomize_rows_action = randomize_rows_action
+          @dedupe_action = dedupe_action
         end
 
         def run
@@ -25,9 +27,13 @@ module Csvtool
             when "2"
               @extract_rows_action.call
             when "3"
+              @randomize_rows_action.call
+            when "4"
+              @dedupe_action.call
+            when "5"
               return 0
             else
-              @stdout.puts "Please choose 1, 2, or 3."
+              @stdout.puts "Please choose 1, 2, 3, 4, or 5."
             end
           end
         end

data/lib/csvtool/interface/cli/prompts/headers_present_prompt.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Interface
+    module CLI
+      module Prompts
+        class HeadersPresentPrompt
+          def initialize(stdin:, stdout:)
+            @stdin = stdin
+            @stdout = stdout
+          end
+
+          def call
+            @stdout.print "Headers present? [Y/n]: "
+            answer = @stdin.gets&.strip.to_s.downcase
+            !%w[n no].include?(answer)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/prompts/seed_prompt.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Interface
+    module CLI
+      module Prompts
+        class SeedPrompt
+          INVALID = :invalid
+
+          def initialize(stdin:, stdout:, errors:)
+            @stdin = stdin
+            @stdout = stdout
+            @errors = errors
+          end
+
+          def call
+            @stdout.print "Random seed (optional integer): "
+            raw = @stdin.gets&.strip.to_s
+            return nil if raw.empty?
+            return raw.to_i if /\A-?\d+\z/.match?(raw)
+
+            @errors.invalid_seed
+            INVALID
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/workflows/run_cross_csv_dedupe_workflow.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require "csv"
+require "csvtool/application/use_cases/run_cross_csv_dedupe"
+require "csvtool/interface/cli/errors/presenter"
+require "csvtool/interface/cli/prompts/file_path_prompt"
+require "csvtool/interface/cli/prompts/separator_prompt"
+require "csvtool/interface/cli/prompts/output_destination_prompt"
+require "csvtool/domain/cross_csv_dedupe_session/csv_profile"
+require "csvtool/domain/cross_csv_dedupe_session/column_selector"
+require "csvtool/domain/cross_csv_dedupe_session/key_mapping"
+require "csvtool/domain/cross_csv_dedupe_session/match_options"
+require "csvtool/domain/cross_csv_dedupe_session/cross_csv_dedupe_session"
+require "csvtool/domain/shared/output_destination"
+
+module Csvtool
+  module Interface
+    module CLI
+      module Workflows
+        class RunCrossCsvDedupeWorkflow
+          def initialize(stdin:, stdout:, use_case: Application::UseCases::RunCrossCsvDedupe.new)
+            @stdin = stdin
+            @stdout = stdout
+            @use_case = use_case
+            @errors = Interface::CLI::Errors::Presenter.new(stdout: stdout)
+          end
+
+          def call
+            source_path = Interface::CLI::Prompts::FilePathPrompt.new(stdin: @stdin, stdout: @stdout).call
+            return @errors.file_not_found(source_path) unless File.file?(source_path)
+
+            @stdout.puts "Source CSV separator:"
+            source_col_sep = Interface::CLI::Prompts::SeparatorPrompt.new(stdin: @stdin, stdout: @stdout, errors: @errors).call
+            return if source_col_sep.nil?
+            @stdout.print "Source headers present? [Y/n]: "
+            source_headers_present = !%w[n no].include?(@stdin.gets&.strip.to_s.downcase)
+            source = Domain::CrossCsvDedupeSession::CsvProfile.new(
+              path: source_path,
+              separator: source_col_sep,
+              headers_present: source_headers_present
+            )
+
+            @stdout.print "Reference CSV file path: "
+            reference_path = @stdin.gets&.strip.to_s
+            return @errors.file_not_found(reference_path) unless File.file?(reference_path)
+
+            @stdout.puts "Reference CSV separator:"
+            reference_col_sep = Interface::CLI::Prompts::SeparatorPrompt.new(stdin: @stdin, stdout: @stdout, errors: @errors).call
+            return if reference_col_sep.nil?
+            @stdout.print "Reference headers present? [Y/n]: "
+            reference_headers_present = !%w[n no].include?(@stdin.gets&.strip.to_s.downcase)
+            reference = Domain::CrossCsvDedupeSession::CsvProfile.new(
+              path: reference_path,
+              separator: reference_col_sep,
+              headers_present: reference_headers_present
+            )
+
+            source_selector = prompt_selector("Source", source.headers_present?)
+            return @errors.column_not_found if source_selector.nil?
+            reference_selector = prompt_selector("Reference", reference.headers_present?)
+            return @errors.column_not_found if reference_selector.nil?
+
+            @stdout.print "Trim whitespace before matching? [Y/n]: "
+            trim_whitespace = read_yes_no(default: true)
+            @stdout.print "Case-insensitive matching? [y/N]: "
+            case_insensitive = read_yes_no(default: false)
+
+            key_mapping = Domain::CrossCsvDedupeSession::KeyMapping.new(
+              source_selector: source_selector,
+              reference_selector: reference_selector
+            )
+            match_options = Domain::CrossCsvDedupeSession::MatchOptions.new(
+              trim_whitespace: trim_whitespace,
+              case_insensitive: case_insensitive
+            )
+            session = Domain::CrossCsvDedupeSession::CrossCsvDedupeSession.start(
+              source: source,
+              reference: reference,
+              key_mapping: key_mapping,
+              match_options: match_options
+            )
+
+            output_destination = Interface::CLI::Prompts::OutputDestinationPrompt.new(
+              stdin: @stdin,
+              stdout: @stdout,
+              errors: @errors
+            ).call
+            return if output_destination.nil?
+            session = session.with_output_destination(
+              if output_destination[:mode] == :file
+                Domain::Shared::OutputDestination.file(path: output_destination[:path])
+              else
+                Domain::Shared::OutputDestination.console
+              end
+            )
+
+            result = @use_case.call(
+              session: session,
+              on_header: ->(headers) { print_header(headers, col_sep: session.source.separator) },
+              on_row: ->(fields) { print_row(fields, col_sep: session.source.separator) }
+            )
+            return handle_error(result) unless result.ok?
+
+            @stdout.puts "Wrote output to #{result.data[:output_path]}" if session.output_destination.file?
+            stats = result.data[:stats]
+            @stdout.puts "Summary: source_rows=#{stats[:source_rows]} removed_rows=#{stats[:removed_rows]} kept_rows=#{stats[:kept_rows_count]}"
+            @stdout.puts "No rows removed; no matching keys found." if stats[:removed_rows].zero?
+            @stdout.puts "All source rows were removed by dedupe." if stats[:source_rows].positive? && stats[:kept_rows_count].zero?
+          rescue ArgumentError => e
+            return @errors.empty_output_path if e.message == "file output path cannot be empty"
+
+            raise e
+          end
+
+          private
+
+          def prompt_selector(label, headers_present)
+            if headers_present
+              @stdout.print "#{label} key column name: "
+            else
+              @stdout.print "#{label} key column index (1-based): "
+            end
+            input = @stdin.gets&.strip.to_s
+            Domain::CrossCsvDedupeSession::ColumnSelector.from_input(headers_present: headers_present, input: input)
+          rescue ArgumentError
+            nil
+          end
+
+          def print_header(headers, col_sep:)
+            @stdout.puts
+            @stdout.puts ::CSV.generate_line(headers, row_sep: "", col_sep: col_sep).chomp
+          end
+
+          def print_row(fields, col_sep:)
+            @stdout.puts ::CSV.generate_line(fields, row_sep: "", col_sep: col_sep).chomp
+          end
+
+          def handle_error(result)
+            case result.error
+            when :column_not_found
+              @errors.column_not_found
+            when :could_not_parse_csv
+              @errors.could_not_parse_csv
+            when :cannot_read_file
+              @errors.cannot_read_file(result.data[:path])
+            when :cannot_write_output_file
+              @errors.cannot_write_output_file(result.data[:path], result.data[:error_class])
+            end
+          end
+
+          def read_yes_no(default:)
+            answer = @stdin.gets&.strip.to_s.downcase
+            return default if answer.empty?
+            return true if %w[y yes].include?(answer)
+            return false if %w[n no].include?(answer)
+
+            default
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Csvtool
-  VERSION = "0.2.0.alpha"
+  VERSION = "0.4.0.alpha"
 end

data/test/csvtool/application/use_cases/run_cross_csv_dedupe_test.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require_relative "../../../test_helper"
+require "csvtool/application/use_cases/run_cross_csv_dedupe"
+require "csvtool/domain/cross_csv_dedupe_session/cross_csv_dedupe_session"
+require "csvtool/domain/cross_csv_dedupe_session/csv_profile"
+require "csvtool/domain/cross_csv_dedupe_session/column_selector"
+require "csvtool/domain/cross_csv_dedupe_session/key_mapping"
+require "csvtool/domain/cross_csv_dedupe_session/match_options"
+require "csvtool/domain/shared/output_destination"
+require "tmpdir"
+
+class RunCrossCsvDedupeTest < Minitest::Test
+  def fixture_path(name)
+    File.expand_path("../../../fixtures/#{name}", __dir__)
+  end
+
+  def test_streams_retained_rows_to_callbacks
+    use_case = Csvtool::Application::UseCases::RunCrossCsvDedupe.new
+    headers = nil
+    rows = []
+
+    result = use_case.call(
+      session: build_session(
+        source_path: fixture_path("dedupe_source.csv"),
+        reference_path: fixture_path("dedupe_reference.csv"),
+        source_selector_input: "customer_id",
+        reference_selector_input: "external_id",
+        output_destination: Csvtool::Domain::Shared::OutputDestination.console
+      ),
+      on_header: ->(value) { headers = value },
+      on_row: ->(fields) { rows << fields }
+    )
+
+    assert_equal true, result.ok?
+    assert_equal ["customer_id", "name"], headers
+    assert_equal [%w[1 Alice], %w[3 Cara]], rows
+    assert_equal 5, result.data[:stats][:source_rows]
+    assert_equal 3, result.data[:stats][:removed_rows]
+    assert_equal 2, result.data[:stats][:kept_rows_count]
+  end
+
+  def test_writes_to_file_output_destination
+    use_case = Csvtool::Application::UseCases::RunCrossCsvDedupe.new
+
+    Dir.mktmpdir do |dir|
+      output_path = File.join(dir, "deduped.csv")
+      result = use_case.call(
+        session: build_session(
+          source_path: fixture_path("dedupe_source.csv"),
+          reference_path: fixture_path("dedupe_reference.csv"),
+          source_selector_input: "customer_id",
+          reference_selector_input: "external_id",
+          output_destination: Csvtool::Domain::Shared::OutputDestination.file(path: output_path)
+        )
+      )
+
+      assert_equal true, result.ok?
+      assert_equal output_path, result.data[:output_path]
+      assert_equal "customer_id,name\n1,Alice\n3,Cara\n", File.read(output_path)
+    end
+  end
+
+  def test_returns_column_not_found_when_selector_invalid
+    use_case = Csvtool::Application::UseCases::RunCrossCsvDedupe.new
+
+    result = use_case.call(
+      session: build_session(
+        source_path: fixture_path("dedupe_source.csv"),
+        reference_path: fixture_path("dedupe_reference.csv"),
+        source_selector_input: "missing",
+        reference_selector_input: "external_id",
+        output_destination: Csvtool::Domain::Shared::OutputDestination.console
+      )
+    )
+
+    assert_equal false, result.ok?
+    assert_equal :column_not_found, result.error
+  end
+
+  private
+
+  def build_session(source_path:, reference_path:, source_selector_input:, reference_selector_input:, output_destination:)
+    source = Csvtool::Domain::CrossCsvDedupeSession::CsvProfile.new(
+      path: source_path,
+      separator: ",",
+      headers_present: true
+    )
+    reference = Csvtool::Domain::CrossCsvDedupeSession::CsvProfile.new(
+      path: reference_path,
+      separator: ",",
+      headers_present: true
+    )
+    key_mapping = Csvtool::Domain::CrossCsvDedupeSession::KeyMapping.new(
+      source_selector: Csvtool::Domain::CrossCsvDedupeSession::ColumnSelector.from_input(
+        headers_present: true,
+        input: source_selector_input
+      ),
+      reference_selector: Csvtool::Domain::CrossCsvDedupeSession::ColumnSelector.from_input(
+        headers_present: true,
+        input: reference_selector_input
+      )
+    )
+    match_options = Csvtool::Domain::CrossCsvDedupeSession::MatchOptions.new(
+      trim_whitespace: true,
+      case_insensitive: false
+    )
+
+    Csvtool::Domain::CrossCsvDedupeSession::CrossCsvDedupeSession
+      .start(source: source, reference: reference, key_mapping: key_mapping, match_options: match_options)
+      .with_output_destination(output_destination)
+  end
+end

data/test/csvtool/application/use_cases/run_row_randomization_test.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+require_relative "../../../test_helper"
+require "csvtool/application/use_cases/run_row_randomization"
+require "tmpdir"
+
+class RunRowRandomizationTest < Minitest::Test
+  def test_prints_header_then_all_randomized_rows
+    fixture = File.expand_path("../../../fixtures/sample_people.csv", __dir__)
+    output = StringIO.new
+    input = StringIO.new("#{fixture}\n\n\n\n\n")
+
+    Csvtool::Application::UseCases::RunRowRandomization.new(stdin: input, stdout: output).call
+
+    assert_includes output.string, "CSV file path:"
+    header_index = output.string.index("name,city")
+    assert header_index
+    %w[Alice,London Bob,Paris Cara,Berlin].each do |row|
+      row_index = output.string.index(row)
+      assert row_index
+      assert_operator header_index, :<, row_index
+    end
+  end
+
+  def test_missing_file_shows_friendly_error
+    output = StringIO.new
+    input = StringIO.new("/tmp/does-not-exist.csv\n")
+
+    Csvtool::Application::UseCases::RunRowRandomization.new(stdin: input, stdout: output).call
+
+    assert_includes output.string, "File not found: /tmp/does-not-exist.csv"
+  end
+
+  def test_can_write_randomized_rows_to_file
+    fixture = File.expand_path("../../../fixtures/sample_people.csv", __dir__)
+    output = StringIO.new
+
+    Dir.mktmpdir do |dir|
+      output_path = File.join(dir, "randomized.csv")
+      input = StringIO.new("#{fixture}\n\n\n\n2\n#{output_path}\n")
+
+      Csvtool::Application::UseCases::RunRowRandomization.new(stdin: input, stdout: output).call
+
+      written = File.read(output_path).lines.map(&:strip)
+      assert_equal "name,city", written.first
+      assert_equal ["Alice,London", "Bob,Paris", "Cara,Berlin"].sort, written[1..].sort
+      assert_includes output.string, "Wrote output to #{output_path}"
+    end
+  end
+
+  def test_supports_tsv_separator
+    fixture = File.expand_path("../../../fixtures/sample_people.tsv", __dir__)
+    output = StringIO.new
+    input = StringIO.new("#{fixture}\n2\n\n\n\n")
+
+    Csvtool::Application::UseCases::RunRowRandomization.new(stdin: input, stdout: output).call
+
+    assert_includes output.string, "name\tcity"
+    assert_includes output.string, "Alice\tLondon"
+    assert_includes output.string, "Bob\tParis"
+    assert_includes output.string, "Cara\tBerlin"
+  end
+
+  def test_supports_custom_separator
+    fixture = File.expand_path("../../../fixtures/sample_people_colon.txt", __dir__)
+    output = StringIO.new
+    input = StringIO.new("#{fixture}\n5\n:\n\n\n\n")
+
+    Csvtool::Application::UseCases::RunRowRandomization.new(stdin: input, stdout: output).call
+
+    assert_includes output.string, "name:city"
+    assert_includes output.string, "Alice:London"
+    assert_includes output.string, "Bob:Paris"
+    assert_includes output.string, "Cara:Berlin"
+  end
+
+  def test_headerless_mode_randomizes_all_rows
+    fixture = File.expand_path("../../../fixtures/sample_people_no_headers.csv", __dir__)
+    output = StringIO.new
+    input = StringIO.new("#{fixture}\n\nn\n\n\n")
+
+    Csvtool::Application::UseCases::RunRowRandomization.new(stdin: input, stdout: output).call
+
+    refute_includes output.string, "name,city"
+    assert_includes output.string, "Alice,London"
+    assert_includes output.string, "Bob,Paris"
+    assert_includes output.string, "Cara,Berlin"
+  end
+
+  def test_same_seed_produces_same_output_order
+    fixture = File.expand_path("../../../fixtures/sample_people_many.csv", __dir__)
+    input_data = "#{fixture}\n\n\n123\n\n"
+
+    out1 = StringIO.new
+    out2 = StringIO.new
+
+    Csvtool::Application::UseCases::RunRowRandomization.new(stdin: StringIO.new(input_data), stdout: out1).call
+    Csvtool::Application::UseCases::RunRowRandomization.new(stdin: StringIO.new(input_data), stdout: out2).call
+
+    rows1 = out1.string.lines.map(&:strip).select { |line| line.include?(",") && !line.start_with?("name,city") }
+    rows2 = out2.string.lines.map(&:strip).select { |line| line.include?(",") && !line.start_with?("name,city") }
+    assert_equal rows1, rows2
+  end
+
+  def test_invalid_seed_shows_friendly_error
+    fixture = File.expand_path("../../../fixtures/sample_people.csv", __dir__)
+    output = StringIO.new
+    input = StringIO.new("#{fixture}\n\n\nabc\n")
+
+    Csvtool::Application::UseCases::RunRowRandomization.new(stdin: input, stdout: output).call
+
+    assert_includes output.string, "Seed must be an integer."
+  end
+
+  def test_malformed_csv_shows_friendly_error
+    fixture = File.expand_path("../../../fixtures/sample_people_bad_tail.csv", __dir__)
+    output = StringIO.new
+    input = StringIO.new("#{fixture}\n\n\n\n\n")
+
+    Csvtool::Application::UseCases::RunRowRandomization.new(stdin: input, stdout: output).call
+
+    assert_includes output.string, "Could not parse CSV file."
+  end
+end