csvops 0.7.0.alpha → 0.9.0.alpha

data/test/csvtool/cli_unit_test.rb

@@ -16,7 +16,7 @@ class CliUnitTest < Minitest::Test
   end
 
   def test_menu_command_can_exit_zero
-    status = Csvtool::CLI.start(["menu"], stdin: StringIO.new("6\n"), stdout: StringIO.new, stderr: StringIO.new)
+    status = Csvtool::CLI.start(["menu"], stdin: StringIO.new("8\n"), stdout: StringIO.new, stderr: StringIO.new)
     assert_equal 0, status
   end
 
@@ -28,7 +28,7 @@ class CliUnitTest < Minitest::Test
   def test_menu_routes_to_row_range_shell
     stdout = StringIO.new
     fixture = File.expand_path("../fixtures/sample_people.csv", __dir__)
-    input = ["2", fixture, "", "2", "3", "", "6"].join("\n") + "\n"
+    input = ["2", fixture, "", "2", "3", "", "8"].join("\n") + "\n"
     status = Csvtool::CLI.start(["menu"], stdin: StringIO.new(input), stdout: stdout, stderr: StringIO.new)
     assert_equal 0, status
     assert_includes stdout.string, "name,city"
@@ -39,7 +39,7 @@ class CliUnitTest < Minitest::Test
   def test_menu_routes_to_randomize_rows_shell
     stdout = StringIO.new
     fixture = File.expand_path("../fixtures/sample_people.csv", __dir__)
-    input = ["3", fixture, "", "", "", "", "6"].join("\n") + "\n"
+    input = ["3", fixture, "", "", "", "", "8"].join("\n") + "\n"
     status = Csvtool::CLI.start(["menu"], stdin: StringIO.new(input), stdout: stdout, stderr: StringIO.new)
     assert_equal 0, status
     assert_includes stdout.string, "name,city"
@@ -52,12 +52,12 @@ class CliUnitTest < Minitest::Test
     stdout = StringIO.new
     source_fixture = File.expand_path("../fixtures/dedupe_source.csv", __dir__)
     reference_fixture = File.expand_path("../fixtures/dedupe_reference.csv", __dir__)
-    input = ["4", source_fixture, "", "", reference_fixture, "", "", "customer_id", "external_id", "", "", "", "6"].join("\n") + "\n"
+    input = ["4", source_fixture, "", "", reference_fixture, "", "", "customer_id", "external_id", "", "", "", "8"].join("\n") + "\n"
     status = Csvtool::CLI.start(["menu"], stdin: StringIO.new(input), stdout: stdout, stderr: StringIO.new)
     assert_equal 0, status
     assert_includes stdout.string, "customer_id,name"
     assert_includes stdout.string, "1,Alice"
     assert_includes stdout.string, "3,Cara"
-    assert_includes stdout.string, "Summary: source_rows=5 removed_rows=3 kept_rows=2"
+    assert_includes stdout.string, "Summary"
   end
 end

data/test/csvtool/infrastructure/csv/csv_stats_scanner_test.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require_relative "../../../test_helper"
+require "csv"
+require "csvtool/infrastructure/csv/csv_stats_scanner"
+require "tmpdir"
+
+class CsvStatsScannerTest < Minitest::Test
+  def fixture_path(name)
+    File.expand_path("../../../fixtures/#{name}", __dir__)
+  end
+
+  def test_scans_headers_mode_with_streaming_foreach
+    source = fixture_path("sample_people_blanks.csv")
+    csv = Object.new
+    received = nil
+
+    define_singleton_foreach(csv) do |path, headers:, col_sep:, &block|
+      received = { path: path, headers: headers, col_sep: col_sep }
+      ::CSV.foreach(path, headers: headers, col_sep: col_sep, &block)
+    end
+
+    result = Csvtool::Infrastructure::CSV::CsvStatsScanner.new(csv: csv).call(
+      file_path: source,
+      col_sep: ",",
+      headers_present: true
+    )
+
+    assert_equal({ path: source, headers: true, col_sep: "," }, received)
+    assert_equal 5, result[:row_count]
+    assert_equal 2, result[:column_count]
+    assert_equal ["name", "city"], result[:headers]
+    assert_equal [
+      { name: "name", blank_count: 2, non_blank_count: 3 },
+      { name: "city", blank_count: 1, non_blank_count: 4 }
+    ], result[:column_stats]
+  end
+
+  def test_scans_large_file_in_single_pass_shape
+    Dir.mktmpdir do |dir|
+      path = File.join(dir, "large.csv")
+      File.open(path, "w") do |f|
+        f.puts("id,value")
+        20_000.times { |i| f.puts("#{i},v#{i}") }
+      end
+
+      result = Csvtool::Infrastructure::CSV::CsvStatsScanner.new.call(
+        file_path: path,
+        col_sep: ",",
+        headers_present: true
+      )
+
+      assert_equal 20_000, result[:row_count]
+      assert_equal 2, result[:column_count]
+      assert_equal ["id", "value"], result[:headers]
+      assert_equal [
+        { name: "id", blank_count: 0, non_blank_count: 20_000 },
+        { name: "value", blank_count: 0, non_blank_count: 20_000 }
+      ], result[:column_stats]
+    end
+  end
+
+  private
+
+  def define_singleton_foreach(obj, &implementation)
+    obj.define_singleton_method(:foreach, &implementation)
+  end
+end

data/test/csvtool/infrastructure/output/csv_stats_file_writer_test.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative "../../../test_helper"
+require "csvtool/infrastructure/output/csv_stats_file_writer"
+require "tmpdir"
+
+class InfrastructureCsvStatsFileWriterTest < Minitest::Test
+  def test_writes_stats_as_metric_value_csv
+    writer = Csvtool::Infrastructure::Output::CsvStatsFileWriter.new
+
+    Dir.mktmpdir do |dir|
+      path = File.join(dir, "stats.csv")
+      writer.call(
+        path: path,
+        data: {
+          row_count: 3,
+          column_count: 2,
+          headers: ["name", "city"],
+          column_stats: [
+            { name: "name", non_blank_count: 3, blank_count: 0 },
+            { name: "city", non_blank_count: 2, blank_count: 1 }
+          ]
+        }
+      )
+
+      assert_equal [
+        "metric,value",
+        "row_count,3",
+        "column_count,2",
+        "headers,name|city",
+        "column.name.non_blank,3",
+        "column.name.blank,0",
+        "column.city.non_blank,2",
+        "column.city.blank,1"
+      ], File.read(path).lines.map(&:strip)
+    end
+  end
+end

data/test/csvtool/interface/cli/menu_loop_test.rb

@@ -17,7 +17,7 @@ class MenuLoopTest < Minitest::Test
   end
 
   def test_routes_extract_column_then_exit
-    menu, actions, = build_menu("1\n7\n")
+    menu, actions, = build_menu("1\n8\n")
     status = menu.run
 
     assert_equal 0, status
@@ -27,10 +27,11 @@ class MenuLoopTest < Minitest::Test
     assert_equal 0, actions[:dedupe].runs
     assert_equal 0, actions[:parity].runs
     assert_equal 0, actions[:split].runs
+    assert_equal 0, actions[:stats].runs
   end
 
   def test_routes_extract_rows_then_exit
-    menu, actions, = build_menu("2\n7\n")
+    menu, actions, = build_menu("2\n8\n")
     status = menu.run
 
     assert_equal 0, status
@@ -40,10 +41,11 @@ class MenuLoopTest < Minitest::Test
     assert_equal 0, actions[:dedupe].runs
     assert_equal 0, actions[:parity].runs
     assert_equal 0, actions[:split].runs
+    assert_equal 0, actions[:stats].runs
   end
 
   def test_routes_randomize_rows_then_exit
-    menu, actions, = build_menu("3\n7\n")
+    menu, actions, = build_menu("3\n8\n")
     status = menu.run
 
     assert_equal 0, status
@@ -53,10 +55,11 @@ class MenuLoopTest < Minitest::Test
     assert_equal 0, actions[:dedupe].runs
     assert_equal 0, actions[:parity].runs
     assert_equal 0, actions[:split].runs
+    assert_equal 0, actions[:stats].runs
   end
 
   def test_routes_dedupe_then_exit
-    menu, actions, = build_menu("4\n7\n")
+    menu, actions, = build_menu("4\n8\n")
     status = menu.run
 
     assert_equal 0, status
@@ -66,10 +69,11 @@ class MenuLoopTest < Minitest::Test
     assert_equal 1, actions[:dedupe].runs
     assert_equal 0, actions[:parity].runs
     assert_equal 0, actions[:split].runs
+    assert_equal 0, actions[:stats].runs
   end
 
   def test_routes_parity_then_exit
-    menu, actions, = build_menu("5\n7\n")
+    menu, actions, = build_menu("5\n8\n")
     status = menu.run
 
     assert_equal 0, status
@@ -79,10 +83,11 @@ class MenuLoopTest < Minitest::Test
     assert_equal 0, actions[:dedupe].runs
     assert_equal 1, actions[:parity].runs
     assert_equal 0, actions[:split].runs
+    assert_equal 0, actions[:stats].runs
   end
 
   def test_routes_split_then_exit
-    menu, actions, stdout = build_menu("6\n7\n")
+    menu, actions, stdout = build_menu("6\n8\n")
     status = menu.run
 
     assert_equal 0, status
@@ -92,20 +97,36 @@ class MenuLoopTest < Minitest::Test
     assert_equal 0, actions[:dedupe].runs
     assert_equal 0, actions[:parity].runs
     assert_equal 1, actions[:split].runs
+    assert_equal 0, actions[:stats].runs
     assert_includes stdout.string, "CSV Tool Menu"
   end
 
+  def test_routes_stats_then_exit
+    menu, actions, = build_menu("7\n8\n")
+    status = menu.run
+
+    assert_equal 0, status
+    assert_equal 0, actions[:column].runs
+    assert_equal 0, actions[:rows].runs
+    assert_equal 0, actions[:randomize].runs
+    assert_equal 0, actions[:dedupe].runs
+    assert_equal 0, actions[:parity].runs
+    assert_equal 0, actions[:split].runs
+    assert_equal 1, actions[:stats].runs
+  end
+
   def test_invalid_choice_shows_prompt
-    menu, actions, stdout = build_menu("x\n7\n")
+    menu, actions, stdout = build_menu("x\n8\n")
     menu.run
 
-    assert_includes stdout.string, "Please choose 1, 2, 3, 4, 5, 6, or 7."
+    assert_includes stdout.string, "Please choose 1, 2, 3, 4, 5, 6, 7, or 8."
     assert_equal 0, actions[:column].runs
     assert_equal 0, actions[:rows].runs
     assert_equal 0, actions[:randomize].runs
     assert_equal 0, actions[:dedupe].runs
     assert_equal 0, actions[:parity].runs
     assert_equal 0, actions[:split].runs
+    assert_equal 0, actions[:stats].runs
   end
 
   private
@@ -117,20 +138,22 @@ class MenuLoopTest < Minitest::Test
       randomize: FakeAction.new,
       dedupe: FakeAction.new,
       parity: FakeAction.new,
-      split: FakeAction.new
+      split: FakeAction.new,
+      stats: FakeAction.new
     }
     stdout = StringIO.new
 
     menu = Csvtool::Interface::CLI::MenuLoop.new(
       stdin: StringIO.new(input),
       stdout: stdout,
-      menu_options: ["Extract column", "Extract rows (range)", "Randomize rows", "Dedupe using another CSV", "Validate parity", "Split CSV into chunks", "Exit"],
+      menu_options: ["Extract column", "Extract rows (range)", "Randomize rows", "Dedupe using another CSV", "Validate parity", "Split CSV into chunks", "CSV stats summary", "Exit"],
       extract_column_action: actions[:column],
       extract_rows_action: actions[:rows],
       randomize_rows_action: actions[:randomize],
       dedupe_action: actions[:dedupe],
       parity_action: actions[:parity],
-      split_action: actions[:split]
+      split_action: actions[:split],
+      stats_action: actions[:stats]
     )
 
     [menu, actions, stdout]

data/test/csvtool/interface/cli/output/color_policy_test.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require_relative "../../../../test_helper"
+require "csvtool/interface/cli/output/color_policy"
+
+class ColorPolicyTest < Minitest::Test
+  class TtyIO
+    def tty? = true
+  end
+
+  class NonTtyIO
+    def tty? = false
+  end
+
+  def test_auto_uses_tty
+    enabled = Csvtool::Interface::CLI::Output::ColorPolicy.new(mode: "auto", io: TtyIO.new, env: {}).enabled?
+    disabled = Csvtool::Interface::CLI::Output::ColorPolicy.new(mode: "auto", io: NonTtyIO.new, env: {}).enabled?
+
+    assert_equal true, enabled
+    assert_equal false, disabled
+  end
+
+  def test_never_disables_color
+    policy = Csvtool::Interface::CLI::Output::ColorPolicy.new(mode: "never", io: TtyIO.new, env: {})
+
+    assert_equal false, policy.enabled?
+  end
+
+  def test_always_enables_color_even_with_no_color
+    policy = Csvtool::Interface::CLI::Output::ColorPolicy.new(mode: "always", io: NonTtyIO.new, env: { "NO_COLOR" => "1" })
+
+    assert_equal true, policy.enabled?
+  end
+
+  def test_no_color_disables_auto
+    policy = Csvtool::Interface::CLI::Output::ColorPolicy.new(mode: "auto", io: TtyIO.new, env: { "NO_COLOR" => "1" })
+
+    assert_equal false, policy.enabled?
+  end
+end

data/test/csvtool/interface/cli/output/colorizer_test.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_relative "../../../../test_helper"
+require "csvtool/interface/cli/output/colorizer"
+
+class ColorizerTest < Minitest::Test
+  class FakePolicy
+    def initialize(enabled)
+      @enabled = enabled
+    end
+
+    def enabled?
+      @enabled
+    end
+  end
+
+  def test_wraps_text_when_enabled
+    colorizer = Csvtool::Interface::CLI::Output::Colorizer.new(policy: FakePolicy.new(true))
+
+    assert_equal "\e[31mMISMATCH\e[0m", colorizer.call("MISMATCH", code: "31")
+  end
+
+  def test_returns_text_when_disabled
+    colorizer = Csvtool::Interface::CLI::Output::Colorizer.new(policy: FakePolicy.new(false))
+
+    assert_equal "MATCH", colorizer.call("MATCH", code: "32")
+  end
+end

data/test/csvtool/interface/cli/output/formatters/csv_row_formatter_test.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require_relative "../../../../../test_helper"
+require "csvtool/interface/cli/output/formatters/csv_row_formatter"
+
+class CsvRowFormatterTest < Minitest::Test
+  def test_formats_row_with_separator
+    formatter = Csvtool::Interface::CLI::Output::Formatters::CsvRowFormatter.new
+
+    row = formatter.call(fields: ["Alice", "London"], col_sep: ",")
+
+    assert_equal "Alice,London", row
+  end
+
+  def test_quotes_values_when_needed
+    formatter = Csvtool::Interface::CLI::Output::Formatters::CsvRowFormatter.new
+
+    row = formatter.call(fields: ["Alice, Jr", "London"], col_sep: ",")
+
+    assert_equal '"Alice, Jr",London', row
+  end
+end

data/test/csvtool/interface/cli/output/formatters/stats_formatter_test.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require_relative "../../../../../test_helper"
+require "csvtool/interface/cli/output/formatters/stats_formatter"
+require "csvtool/interface/cli/output/table_renderer"
+require "json"
+
+class StatsFormatterTest < Minitest::Test
+  def sample_data
+    {
+      row_count: 3,
+      column_count: 2,
+      headers: ["name", "city"],
+      column_stats: [
+        { name: "name", non_blank_count: 3, blank_count: 0 },
+        { name: "city", non_blank_count: 2, blank_count: 1 }
+      ]
+    }
+  end
+
+  def formatter
+    Csvtool::Interface::CLI::Output::Formatters::StatsFormatter.new(
+      table_renderer: Csvtool::Interface::CLI::Output::TableRenderer.new
+    )
+  end
+
+  def test_formats_text_summary
+    text = formatter.call(data: sample_data, format: "text", max_width: 80)
+
+    assert_includes text, "CSV Stats Summary"
+    assert_includes text, "Metric"
+    assert_includes text, "Column completeness:"
+  end
+
+  def test_formats_json_summary
+    json = formatter.call(data: sample_data, format: "json", max_width: 80)
+
+    parsed = JSON.parse(json, symbolize_names: true)
+    assert_equal 3, parsed[:row_count]
+    assert_equal 2, parsed[:column_count]
+  end
+
+  def test_formats_csv_summary
+    csv = formatter.call(data: sample_data, format: "csv", max_width: 80)
+
+    lines = csv.lines.map(&:chomp)
+    assert_equal "metric,value", lines.first
+    assert_includes lines, "row_count,3"
+    assert_includes lines, "column.city.blank,1"
+  end
+end

data/test/csvtool/interface/cli/output/streams_test.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative "../../../../test_helper"
+require "csvtool/interface/cli/output/streams"
+
+class StreamsTest < Minitest::Test
+  def test_builds_data_and_ui_streams
+    data = StringIO.new
+    ui = StringIO.new
+
+    streams = Csvtool::Interface::CLI::Output::Streams.build(data: data, ui: ui)
+
+    assert_same data, streams.data
+    assert_same ui, streams.ui
+  end
+
+  def test_defaults_ui_to_data_stream
+    data = StringIO.new
+
+    streams = Csvtool::Interface::CLI::Output::Streams.build(data: data)
+
+    assert_same data, streams.data
+    assert_same data, streams.ui
+  end
+end

data/test/csvtool/interface/cli/output/table_renderer_test.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require_relative "../../../../test_helper"
+require "csvtool/interface/cli/output/table_renderer"
+
+class TableRendererTest < Minitest::Test
+  def test_renders_aligned_table
+    renderer = Csvtool::Interface::CLI::Output::TableRenderer.new
+
+    text = renderer.render(
+      headers: ["Metric", "Value"],
+      rows: [["Rows", "3"], ["Columns", "2"]],
+      max_width: 80
+    )
+
+    lines = text.lines.map(&:chomp)
+    assert_equal "Metric  | Value", lines[0]
+    assert_equal "--------+------", lines[1]
+    assert_equal "Rows    | 3    ", lines[2]
+    assert_equal "Columns | 2    ", lines[3]
+  end
+
+  def test_truncates_cells_when_width_is_narrow
+    renderer = Csvtool::Interface::CLI::Output::TableRenderer.new
+
+    text = renderer.render(
+      headers: ["Column", "Non-blank", "Blank"],
+      rows: [["very_long_column_name", "123456", "0"]],
+      max_width: 26
+    )
+
+    lines = text.lines.map(&:chomp)
+    assert lines.all? { |line| line.length <= 26 }
+    assert_includes lines[2], "..."
+  end
+end

data/test/csvtool/interface/cli/workflows/builders/csv_stats_session_builder_test.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative "../../../../../test_helper"
+require "csvtool/interface/cli/workflows/builders/csv_stats_session_builder"
+require "csvtool/domain/shared/output_destination"
+
+class CsvStatsSessionBuilderTest < Minitest::Test
+  def test_builds_stats_session
+    builder = Csvtool::Interface::CLI::Workflows::Builders::CsvStatsSessionBuilder.new
+    destination = Csvtool::Domain::Shared::OutputDestination.console
+
+    session = builder.call(file_path: "/tmp/data.csv", col_sep: ";", headers_present: false, destination: destination)
+
+    assert_equal "/tmp/data.csv", session.source.path
+    assert_equal ";", session.source.separator
+    assert_equal false, session.source.headers_present
+    assert_equal true, session.output_destination.console?
+  end
+end

data/test/csvtool/interface/cli/workflows/presenters/cross_csv_dedupe_presenter_test.rb

@@ -14,7 +14,10 @@ class CrossCsvDedupePresenterTest < Minitest::Test
 
     assert_includes out.string, "\nid,name\n"
     assert_includes out.string, "1,Alice"
-    assert_includes out.string, "Summary: source_rows=5 removed_rows=3 kept_rows=2"
+    assert_includes out.string, "Summary"
+    assert_includes out.string, "Source rows"
+    assert_includes out.string, "Removed rows"
+    assert_includes out.string, "Kept rows"
   end
 
   def test_prints_zero_and_all_removed_messages

data/test/csvtool/interface/cli/workflows/presenters/csv_parity_presenter_test.rb

@@ -17,7 +17,11 @@ class CsvParityPresenterTest < Minitest::Test
     )
 
     assert_includes out.string, "MATCH"
-    assert_includes out.string, "Summary: left_rows=3 right_rows=3 left_only=0 right_only=0"
+    assert_includes out.string, "Metric"
+    assert_includes out.string, "Left rows"
+    assert_includes out.string, "Right rows"
+    assert_includes out.string, "Left only"
+    assert_includes out.string, "Right only"
   end
 
   def test_prints_mismatch_examples

data/test/csvtool/interface/cli/workflows/presenters/csv_split_presenter_test.rb

@@ -17,10 +17,28 @@ class CsvSplitPresenterTest < Minitest::Test
     )
 
     assert_includes out.string, "Split complete."
-    assert_includes out.string, "Chunk size: 10"
-    assert_includes out.string, "Data rows: 25"
-    assert_includes out.string, "Chunks written: 3"
-    assert_includes out.string, "Manifest: /tmp/manifest.csv"
+    assert_includes out.string, "Metric"
+    assert_includes out.string, "Chunk size"
+    assert_includes out.string, "Data rows"
+    assert_includes out.string, "Chunks written"
+    assert_includes out.string, "Manifest"
     assert_includes out.string, "/tmp/people_part_001.csv"
   end
+
+  def test_truncates_summary_table_for_narrow_width
+    out = StringIO.new
+    presenter = Csvtool::Interface::CLI::Workflows::Presenters::CsvSplitPresenter.new(stdout: out, max_width: 26)
+
+    presenter.print_summary(
+      chunk_size: 10,
+      data_rows: 25,
+      chunk_count: 3,
+      manifest_path: "/tmp/very/long/path/manifest.csv",
+      chunk_paths: []
+    )
+
+    lines = out.string.lines.map(&:chomp)
+    assert lines.all? { |line| line.empty? || line.length <= 26 }
+    assert_includes out.string, "..."
+  end
 end

data/test/csvtool/interface/cli/workflows/presenters/csv_stats_presenter_test.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative "../../../../../test_helper"
+require "csvtool/interface/cli/workflows/presenters/csv_stats_presenter"
+
+class CsvStatsPresenterTest < Minitest::Test
+  def test_prints_summary_with_headers_and_column_stats
+    out = StringIO.new
+    presenter = Csvtool::Interface::CLI::Workflows::Presenters::CsvStatsPresenter.new(stdout: out)
+
+    presenter.print_summary(
+      row_count: 3,
+      column_count: 2,
+      headers: ["name", "city"],
+      column_stats: [
+        { name: "name", non_blank_count: 3, blank_count: 0 },
+        { name: "city", non_blank_count: 2, blank_count: 1 }
+      ]
+    )
+
+    assert_includes out.string, "CSV Stats Summary"
+    assert_includes out.string, "Metric"
+    assert_includes out.string, "Rows"
+    assert_includes out.string, "Columns"
+    assert_includes out.string, "Headers"
+    assert_includes out.string, "Column completeness:"
+    assert_includes out.string, "name"
+    assert_includes out.string, "city"
+  end
+
+  def test_prints_file_written_message
+    out = StringIO.new
+    presenter = Csvtool::Interface::CLI::Workflows::Presenters::CsvStatsPresenter.new(stdout: out)
+
+    presenter.print_file_written("/tmp/stats.csv")
+
+    assert_includes out.string, "Wrote output to /tmp/stats.csv"
+  end
+end