csvops 0.7.0.alpha → 0.9.0.alpha

data/lib/csvtool/cli.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "csv"
+require "json"
 require "csvtool/interface/cli/menu_loop"
 require "csvtool/interface/cli/workflows/run_extraction_workflow"
 require "csvtool/interface/cli/workflows/run_row_extraction_workflow"
@@ -8,10 +9,21 @@ require "csvtool/interface/cli/workflows/run_row_randomization_workflow"
 require "csvtool/interface/cli/workflows/run_cross_csv_dedupe_workflow"
 require "csvtool/interface/cli/workflows/run_csv_parity_workflow"
 require "csvtool/interface/cli/workflows/run_csv_split_workflow"
+require "csvtool/interface/cli/workflows/run_csv_stats_workflow"
 require "csvtool/interface/cli/errors/presenter"
+require "csvtool/interface/cli/output/table_renderer"
+require "csvtool/interface/cli/output/streams"
+require "csvtool/interface/cli/output/color_policy"
+require "csvtool/interface/cli/output/colorizer"
+require "csvtool/interface/cli/output/formatters/stats_formatter"
 require "csvtool/infrastructure/csv/header_reader"
 require "csvtool/infrastructure/csv/value_streamer"
 require "csvtool/infrastructure/output/console_writer"
+require "csvtool/application/use_cases/run_csv_stats"
+require "csvtool/domain/csv_stats_session/stats_source"
+require "csvtool/domain/csv_stats_session/stats_options"
+require "csvtool/domain/csv_stats_session/stats_session"
+require "csvtool/domain/shared/output_destination"
 
 module Csvtool
   class CLI
@@ -22,18 +34,20 @@ module Csvtool
       "Dedupe using another CSV",
       "Validate parity",
       "Split CSV into chunks",
+      "CSV stats summary",
       "Exit"
     ].freeze
 
-    def self.start(argv, stdin:, stdout:, stderr:)
-      new(argv, stdin: stdin, stdout: stdout, stderr: stderr).run
+    def self.start(argv, stdin:, stdout:, stderr:, env: ENV)
+      new(argv, stdin: stdin, stdout: stdout, stderr: stderr, env: env).run
     end
 
-    def initialize(argv, stdin:, stdout:, stderr:)
+    def initialize(argv, stdin:, stdout:, stderr:, env: ENV)
       @argv = argv
       @stdin = stdin
       @stdout = stdout
       @stderr = stderr
+      @env = env
     end
 
     def run
@@ -42,6 +56,8 @@ module Csvtool
         run_menu_loop
       when "column"
         run_column_command
+      when "stats"
+        run_stats_command
       else
         print_usage
         1
@@ -51,22 +67,26 @@ module Csvtool
     private
 
     def run_menu_loop
-      extract_column_action = -> { Interface::CLI::Workflows::RunExtractionWorkflow.new(stdin: @stdin, stdout: @stdout).call }
-      extract_rows_action = -> { Interface::CLI::Workflows::RunRowExtractionWorkflow.new(stdin: @stdin, stdout: @stdout).call }
-      randomize_rows_action = -> { Interface::CLI::Workflows::RunRowRandomizationWorkflow.new(stdin: @stdin, stdout: @stdout).call }
-      dedupe_action = -> { Interface::CLI::Workflows::RunCrossCsvDedupeWorkflow.new(stdin: @stdin, stdout: @stdout).call }
-      parity_action = -> { Interface::CLI::Workflows::RunCsvParityWorkflow.new(stdin: @stdin, stdout: @stdout).call }
-      split_action = -> { Interface::CLI::Workflows::RunCsvSplitWorkflow.new(stdin: @stdin, stdout: @stdout).call }
+      streams = Interface::CLI::Output::Streams.build(data: @stdout, ui: @stderr)
+      extract_column_action = -> { Interface::CLI::Workflows::RunExtractionWorkflow.new(stdin: @stdin, stdout: streams.data, stderr: streams.ui).call }
+      extract_rows_action = -> { Interface::CLI::Workflows::RunRowExtractionWorkflow.new(stdin: @stdin, stdout: streams.data, stderr: streams.ui).call }
+      randomize_rows_action = -> { Interface::CLI::Workflows::RunRowRandomizationWorkflow.new(stdin: @stdin, stdout: streams.data, stderr: streams.ui).call }
+      dedupe_action = -> { Interface::CLI::Workflows::RunCrossCsvDedupeWorkflow.new(stdin: @stdin, stdout: streams.data, stderr: streams.ui).call }
+      parity_action = -> { Interface::CLI::Workflows::RunCsvParityWorkflow.new(stdin: @stdin, stdout: streams.data, stderr: streams.ui).call }
+      split_action = -> { Interface::CLI::Workflows::RunCsvSplitWorkflow.new(stdin: @stdin, stdout: streams.data, stderr: streams.ui).call }
+      stats_action = -> { Interface::CLI::Workflows::RunCsvStatsWorkflow.new(stdin: @stdin, stdout: streams.data, stderr: streams.ui).call }
       Interface::CLI::MenuLoop.new(
         stdin: @stdin,
-        stdout: @stdout,
+        stdout: streams.data,
+        stderr: streams.ui,
         menu_options: MENU_OPTIONS,
         extract_column_action: extract_column_action,
         extract_rows_action: extract_rows_action,
         randomize_rows_action: randomize_rows_action,
         dedupe_action: dedupe_action,
         parity_action: parity_action,
-        split_action: split_action
+        split_action: split_action,
+        stats_action: stats_action
       ).run
     end
 
@@ -74,6 +94,7 @@ module Csvtool
       @stderr.puts "Usage:"
       @stderr.puts "  csvtool menu"
       @stderr.puts "  csvtool column <file> <column>"
+      @stderr.puts "  csvtool stats <file> [--format text|json|csv] [--color auto|always|never]"
     end
 
     def run_column_command
@@ -84,7 +105,7 @@ module Csvtool
         return 1
       end
 
-      errors = Interface::CLI::Errors::Presenter.new(stdout: @stdout)
+      errors = Interface::CLI::Errors::Presenter.new(stdout: @stderr)
       return errors.file_not_found(file_path) || 1 unless File.file?(file_path)
 
       header_reader = Infrastructure::CSV::HeaderReader.new
@@ -103,5 +124,108 @@ module Csvtool
       errors.cannot_read_file(file_path)
       1
     end
+
+    def run_stats_command
+      file_path, format, color_mode = parse_stats_args(@argv[1..])
+      unless file_path
+        print_usage
+        return 1
+      end
+
+      errors = Interface::CLI::Errors::Presenter.new(stdout: @stderr)
+      unless %w[text json csv].include?(format)
+        @stderr.puts "Invalid format: #{format}"
+        return 1
+      end
+      unless %w[auto always never].include?(color_mode)
+        @stderr.puts "Invalid color mode: #{color_mode}"
+        return 1
+      end
+      source = Domain::CsvStatsSession::StatsSource.new(path: file_path, separator: ",", headers_present: true)
+      options = Domain::CsvStatsSession::StatsOptions.new
+      destination = Domain::Shared::OutputDestination.console
+      session = Domain::CsvStatsSession::StatsSession.start(source: source, options: options).with_output_destination(destination)
+      result = Application::UseCases::RunCsvStats.new.call(session: session)
+
+      unless result.ok?
+        case result.error
+        when :file_not_found
+          errors.file_not_found(result.data[:path])
+        when :could_not_parse_csv
+          errors.could_not_parse_csv
+        when :cannot_read_file
+          errors.cannot_read_file(result.data[:path])
+        else
+          @stderr.puts "Unknown error."
+        end
+        return 1
+      end
+
+      formatter = Interface::CLI::Output::Formatters::StatsFormatter.new(
+        table_renderer: Interface::CLI::Output::TableRenderer.new
+      )
+      output = formatter.call(data: result.data, format: format, max_width: terminal_width)
+      print_stats_output(output, format: format, color_mode: color_mode)
+
+      0
+    end
+
+    def parse_stats_args(args)
+      file_path = args[0]
+      format = "text"
+      color_mode = "auto"
+      index = 1
+      while index < args.length
+        arg = args[index]
+        if arg.start_with?("--format=")
+          format = arg.split("=", 2)[1]
+        elsif arg == "--format"
+          format = args[index + 1].to_s
+          index += 1
+        elsif arg.start_with?("--color=")
+          color_mode = arg.split("=", 2)[1]
+        elsif arg == "--color"
+          color_mode = args[index + 1].to_s
+          index += 1
+        end
+        index += 1
+      end
+      [file_path, format, color_mode]
+    end
+
+    def print_stats_output(output, format:, color_mode:)
+      if format == "text"
+        policy = Interface::CLI::Output::ColorPolicy.new(mode: color_mode, io: @stdout, env: @env)
+        colorizer = Interface::CLI::Output::Colorizer.new(policy: policy)
+        text = apply_text_color(output, colorizer: colorizer)
+        @stdout.puts text
+      else
+        @stdout.puts output
+      end
+    end
+
+    def apply_text_color(text, colorizer:)
+      text.lines.map do |line|
+        line = line.chomp
+        case line
+        when "CSV Stats Summary"
+          colorizer.call(line, code: "1;36")
+        when "Column completeness:"
+          colorizer.call(line, code: "1")
+        when /\A(Metric|Value|Column|Non-blank|Blank)(\s+\|.*)?\z/
+          colorizer.call(line, code: "1")
+        else
+          line
+        end
+      end.join("\n")
+    end
+
+    def terminal_width
+      columns = @env["COLUMNS"].to_i
+      return columns if columns.positive?
+      return @stdout.winsize[1] if @stdout.respond_to?(:winsize)
+
+      80
+    end
   end
 end

data/lib/csvtool/domain/csv_stats_session/stats_options.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Domain
+    module CsvStatsSession
+      class StatsOptions
+        def initialize; end
+      end
+    end
+  end
+end

data/lib/csvtool/domain/csv_stats_session/stats_session.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Domain
+    module CsvStatsSession
+      class StatsSession
+        attr_reader :source, :options, :output_destination
+
+        def self.start(source:, options:)
+          new(source: source, options: options)
+        end
+
+        def initialize(source:, options:, output_destination: nil)
+          @source = source
+          @options = options
+          @output_destination = output_destination
+        end
+
+        def with_output_destination(output_destination)
+          self.class.new(source: source, options: options, output_destination: output_destination)
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/domain/csv_stats_session/stats_source.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Domain
+    module CsvStatsSession
+      class StatsSource
+        attr_reader :path, :separator, :headers_present
+
+        def initialize(path:, separator:, headers_present:)
+          @path = path
+          @separator = separator
+          @headers_present = headers_present
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/infrastructure/csv/csv_stats_scanner.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Csvtool
+  module Infrastructure
+    module CSV
+      class CsvStatsScanner
+        def initialize(csv: ::CSV)
+          @csv = csv
+        end
+
+        def call(file_path:, col_sep:, headers_present:)
+          data_row_count = 0
+          headers = nil
+          column_count = 0
+          column_stats = []
+
+          # Streaming scan: memory grows with per-column metrics, not row count.
+          @csv.foreach(file_path, headers: headers_present, col_sep: col_sep) do |row|
+            if headers_present
+              headers ||= row.headers
+              column_count = headers.length
+              if column_stats.empty?
+                column_stats = headers.map { |name| { name: name, blank_count: 0, non_blank_count: 0 } }
+              end
+              fields = row.fields
+              fields.fill(nil, fields.length...column_count)
+              fields.each_with_index { |value, index| apply_value(column_stats[index], value) }
+              data_row_count += 1
+            else
+              fields = row.is_a?(::CSV::Row) ? row.fields : row
+              column_count = [column_count, fields.length].max
+              while column_stats.length < column_count
+                column_stats << {
+                  name: "column_#{column_stats.length + 1}",
+                  blank_count: 0,
+                  non_blank_count: 0
+                }
+              end
+              fields.fill(nil, fields.length...column_count)
+              fields.each_with_index { |value, index| apply_value(column_stats[index], value) }
+              data_row_count += 1
+            end
+          end
+
+          {
+            row_count: data_row_count,
+            column_count: column_count,
+            headers: headers,
+            column_stats: column_stats
+          }
+        end
+
+        private
+
+        def apply_value(stats, value)
+          if value.nil? || value.strip.empty?
+            stats[:blank_count] += 1
+          else
+            stats[:non_blank_count] += 1
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/infrastructure/output/csv_stats_file_writer.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Csvtool
+  module Infrastructure
+    module Output
+      class CsvStatsFileWriter
+        def call(path:, data:)
+          ::CSV.open(path, "w") do |csv|
+            csv << %w[metric value]
+            csv << ["row_count", data[:row_count]]
+            csv << ["column_count", data[:column_count]]
+            unless data[:headers].nil? || data[:headers].empty?
+              csv << ["headers", data[:headers].join("|")]
+            end
+            data.fetch(:column_stats, []).each do |stats|
+              csv << ["column.#{stats[:name]}.non_blank", stats[:non_blank_count]]
+              csv << ["column.#{stats[:name]}.blank", stats[:blank_count]]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/menu_loop.rb

@@ -4,9 +4,10 @@ module Csvtool
   module Interface
     module CLI
       class MenuLoop
-        def initialize(stdin:, stdout:, menu_options:, extract_column_action:, extract_rows_action:, randomize_rows_action:, dedupe_action:, parity_action:, split_action:)
+        def initialize(stdin:, stdout:, stderr: stdout, menu_options:, extract_column_action:, extract_rows_action:, randomize_rows_action:, dedupe_action:, parity_action:, split_action:, stats_action:)
           @stdin = stdin
           @stdout = stdout
+          @stderr = stderr
           @menu_options = menu_options
           @extract_column_action = extract_column_action
           @extract_rows_action = extract_rows_action
@@ -14,12 +15,13 @@ module Csvtool
           @dedupe_action = dedupe_action
           @parity_action = parity_action
           @split_action = split_action
+          @stats_action = stats_action
         end
 
         def run
           loop do
             print_menu
-            @stdout.print "> "
+            @stderr.print "> "
             choice = @stdin.gets
             return 0 if choice.nil?
 
@@ -37,9 +39,11 @@ module Csvtool
             when "6"
               @split_action.call
             when "7"
+              @stats_action.call
+            when "8"
               return 0
             else
-              @stdout.puts "Please choose 1, 2, 3, 4, 5, 6, or 7."
+              @stderr.puts "Please choose 1, 2, 3, 4, 5, 6, 7, or 8."
             end
           end
         end
@@ -47,9 +51,9 @@ module Csvtool
         private
 
         def print_menu
-          @stdout.puts "CSV Tool Menu"
+          @stderr.puts "CSV Tool Menu"
           @menu_options.each_with_index do |option, index|
-            @stdout.puts "#{index + 1}. #{option}"
+            @stderr.puts "#{index + 1}. #{option}"
           end
         end
       end

data/lib/csvtool/interface/cli/output/color_policy.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Interface
+    module CLI
+      module Output
+        class ColorPolicy
+          def initialize(mode:, io:, env: ENV)
+            @mode = mode
+            @io = io
+            @env = env
+          end
+
+          def enabled?
+            return true if @mode == "always"
+            return false if @mode == "never"
+            return false if @env["NO_COLOR"]
+
+            @io.respond_to?(:tty?) && @io.tty?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/output/colorizer.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "csvtool/interface/cli/output/color_policy"
+
+module Csvtool
+  module Interface
+    module CLI
+      module Output
+        class Colorizer
+          def initialize(policy:)
+            @policy = policy
+          end
+
+          def call(text, code:)
+            return text unless @policy.enabled?
+
+            "\e[#{code}m#{text}\e[0m"
+          end
+
+          def self.auto(io:, env: ENV)
+            new(policy: ColorPolicy.new(mode: "auto", io: io, env: env))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/output/formatters/csv_row_formatter.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Csvtool
+  module Interface
+    module CLI
+      module Output
+        module Formatters
+          class CsvRowFormatter
+            def call(fields:, col_sep:)
+              ::CSV.generate_line(fields, row_sep: "", col_sep: col_sep).chomp
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/output/formatters/stats_formatter.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Csvtool
+  module Interface
+    module CLI
+      module Output
+        module Formatters
+          class StatsFormatter
+            def initialize(table_renderer:)
+              @table_renderer = table_renderer
+            end
+
+            def call(data:, format:, max_width: 80)
+              case format
+              when "json"
+                JSON.generate(data)
+              when "csv"
+                csv_lines(data).join("\n")
+              else
+                text_lines(data, max_width: max_width).join("\n")
+              end
+            end
+
+            private
+
+            def csv_lines(data)
+              lines = ["metric,value", "row_count,#{data[:row_count]}", "column_count,#{data[:column_count]}"]
+              lines << "headers,#{data[:headers].join('|')}" unless data[:headers].nil? || data[:headers].empty?
+              data.fetch(:column_stats, []).each do |stats|
+                lines << "column.#{stats[:name]}.non_blank,#{stats[:non_blank_count]}"
+                lines << "column.#{stats[:name]}.blank,#{stats[:blank_count]}"
+              end
+              lines
+            end
+
+            def text_lines(data, max_width:)
+              lines = ["CSV Stats Summary"]
+              summary_rows = [["Rows", data[:row_count].to_s], ["Columns", data[:column_count].to_s]]
+              summary_rows << ["Headers", data[:headers].join(", ")] unless data[:headers].nil? || data[:headers].empty?
+              lines << @table_renderer.render(headers: ["Metric", "Value"], rows: summary_rows, max_width: max_width)
+
+              return lines if data[:column_stats].nil? || data[:column_stats].empty?
+
+              lines << ""
+              lines << "Column completeness:"
+              rows = data[:column_stats].map { |stats| [stats[:name], stats[:non_blank_count].to_s, stats[:blank_count].to_s] }
+              lines << @table_renderer.render(headers: ["Column", "Non-blank", "Blank"], rows: rows, max_width: max_width)
+              lines
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/output/streams.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Interface
+    module CLI
+      module Output
+        class Streams
+          attr_reader :data, :ui
+
+          def self.build(data:, ui: data)
+            new(data: data, ui: ui)
+          end
+
+          def initialize(data:, ui:)
+            @data = data
+            @ui = ui
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/output/table_renderer.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Interface
+    module CLI
+      module Output
+        class TableRenderer
+          MIN_COLUMN_WIDTH = 4
+
+          def render(headers:, rows:, max_width: 80)
+            widths = compute_widths(headers, rows)
+            widths = fit_widths(widths, max_width)
+
+            lines = []
+            lines << format_row(headers, widths)
+            lines << separator(widths)
+            rows.each { |row| lines << format_row(row, widths) }
+            lines.join("\n")
+          end
+
+          private
+
+          def compute_widths(headers, rows)
+            widths = headers.map { |header| header.to_s.length }
+            rows.each do |row|
+              row.each_with_index do |cell, index|
+                widths[index] = [widths[index], cell.to_s.length].max
+              end
+            end
+            widths
+          end
+
+          def fit_widths(widths, max_width)
+            return widths if total_width(widths) <= max_width
+
+            adjusted = widths.dup
+            while total_width(adjusted) > max_width
+              index = adjusted.each_with_index.max_by { |width, _i| width }[1]
+              break if adjusted[index] <= MIN_COLUMN_WIDTH
+
+              adjusted[index] -= 1
+            end
+            adjusted
+          end
+
+          def total_width(widths)
+            widths.sum + (3 * (widths.length - 1))
+          end
+
+          def separator(widths)
+            widths.map { |width| "-" * width }.join("-+-")
+          end
+
+          def format_row(row, widths)
+            row.each_with_index.map do |cell, index|
+              truncate(cell.to_s, widths[index]).ljust(widths[index])
+            end.join(" | ")
+          end
+
+          def truncate(text, width)
+            return text if text.length <= width
+            return text[0, width] if width < MIN_COLUMN_WIDTH
+
+            "#{text[0, width - 3]}..."
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/workflows/builders/csv_stats_session_builder.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "csvtool/domain/csv_stats_session/stats_source"
+require "csvtool/domain/csv_stats_session/stats_options"
+require "csvtool/domain/csv_stats_session/stats_session"
+
+module Csvtool
+  module Interface
+    module CLI
+      module Workflows
+        module Builders
+          class CsvStatsSessionBuilder
+            def call(file_path:, col_sep:, headers_present:, destination:)
+              source = Domain::CsvStatsSession::StatsSource.new(
+                path: file_path,
+                separator: col_sep,
+                headers_present: headers_present
+              )
+              options = Domain::CsvStatsSession::StatsOptions.new
+              session = Domain::CsvStatsSession::StatsSession.start(source: source, options: options)
+              session.with_output_destination(destination)
+            end
+          end
+        end
+      end
+    end
+  end
+end