csvops 0.6.0.alpha → 0.8.0.alpha

data/lib/csvtool/cli.rb

@@ -7,6 +7,8 @@ require "csvtool/interface/cli/workflows/run_row_extraction_workflow"
 require "csvtool/interface/cli/workflows/run_row_randomization_workflow"
 require "csvtool/interface/cli/workflows/run_cross_csv_dedupe_workflow"
 require "csvtool/interface/cli/workflows/run_csv_parity_workflow"
+require "csvtool/interface/cli/workflows/run_csv_split_workflow"
+require "csvtool/interface/cli/workflows/run_csv_stats_workflow"
 require "csvtool/interface/cli/errors/presenter"
 require "csvtool/infrastructure/csv/header_reader"
 require "csvtool/infrastructure/csv/value_streamer"
@@ -20,6 +22,8 @@ module Csvtool
       "Randomize rows",
       "Dedupe using another CSV",
       "Validate parity",
+      "Split CSV into chunks",
+      "CSV stats summary",
       "Exit"
     ].freeze
 
@@ -54,6 +58,8 @@ module Csvtool
       randomize_rows_action = -> { Interface::CLI::Workflows::RunRowRandomizationWorkflow.new(stdin: @stdin, stdout: @stdout).call }
       dedupe_action = -> { Interface::CLI::Workflows::RunCrossCsvDedupeWorkflow.new(stdin: @stdin, stdout: @stdout).call }
       parity_action = -> { Interface::CLI::Workflows::RunCsvParityWorkflow.new(stdin: @stdin, stdout: @stdout).call }
+      split_action = -> { Interface::CLI::Workflows::RunCsvSplitWorkflow.new(stdin: @stdin, stdout: @stdout).call }
+      stats_action = -> { Interface::CLI::Workflows::RunCsvStatsWorkflow.new(stdin: @stdin, stdout: @stdout).call }
       Interface::CLI::MenuLoop.new(
         stdin: @stdin,
         stdout: @stdout,
@@ -62,7 +68,9 @@ module Csvtool
         extract_rows_action: extract_rows_action,
         randomize_rows_action: randomize_rows_action,
         dedupe_action: dedupe_action,
-        parity_action: parity_action
+        parity_action: parity_action,
+        split_action: split_action,
+        stats_action: stats_action
       ).run
     end
 

data/lib/csvtool/domain/csv_split_session/split_options.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Domain
+    module CsvSplitSession
+      class SplitOptions
+        attr_reader :chunk_size, :output_directory, :file_prefix, :overwrite_existing, :write_manifest, :manifest_path
+
+        def initialize(
+          chunk_size:,
+          output_directory: nil,
+          file_prefix: nil,
+          overwrite_existing: false,
+          write_manifest: false,
+          manifest_path: nil
+        )
+          @chunk_size = Integer(chunk_size)
+          @output_directory = output_directory
+          @file_prefix = file_prefix
+          @overwrite_existing = overwrite_existing
+          @write_manifest = write_manifest
+          @manifest_path = manifest_path
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/domain/csv_split_session/split_session.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Domain
+    module CsvSplitSession
+      class SplitSession
+        attr_reader :source, :options
+
+        def self.start(source:, options:)
+          new(source: source, options: options)
+        end
+
+        def initialize(source:, options:)
+          @source = source
+          @options = options
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/domain/csv_split_session/split_source.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Domain
+    module CsvSplitSession
+      class SplitSource
+        attr_reader :path, :separator, :headers_present
+
+        def initialize(path:, separator:, headers_present:)
+          @path = path
+          @separator = separator
+          @headers_present = headers_present
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/domain/csv_stats_session/stats_options.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Domain
+    module CsvStatsSession
+      class StatsOptions
+        def initialize; end
+      end
+    end
+  end
+end

data/lib/csvtool/domain/csv_stats_session/stats_session.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Domain
+    module CsvStatsSession
+      class StatsSession
+        attr_reader :source, :options, :output_destination
+
+        def self.start(source:, options:)
+          new(source: source, options: options)
+        end
+
+        def initialize(source:, options:, output_destination: nil)
+          @source = source
+          @options = options
+          @output_destination = output_destination
+        end
+
+        def with_output_destination(output_destination)
+          self.class.new(source: source, options: options, output_destination: output_destination)
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/domain/csv_stats_session/stats_source.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Domain
+    module CsvStatsSession
+      class StatsSource
+        attr_reader :path, :separator, :headers_present
+
+        def initialize(path:, separator:, headers_present:)
+          @path = path
+          @separator = separator
+          @headers_present = headers_present
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/infrastructure/csv/csv_splitter.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Csvtool
+  module Infrastructure
+    module CSV
+      class CsvSplitter
+        class OutputFileExistsError < StandardError
+          attr_reader :path
+
+          def initialize(path)
+            super("output file exists: #{path}")
+            @path = path
+          end
+        end
+
+        def call(file_path:, col_sep:, headers_present:, chunk_size:, output_directory:, file_prefix:, overwrite_existing:)
+          ext = File.extname(file_path)
+          ext = ".csv" if ext.empty?
+          sequence = 0
+          data_rows = 0
+          chunk_paths = []
+          chunk_row_counts = []
+          rows_in_chunk = 0
+          current_csv = nil
+
+          write_mode_headers = nil
+          write_headers = headers_present
+
+          ::CSV.foreach(file_path, headers: headers_present, col_sep: col_sep) do |row|
+            if current_csv.nil? || rows_in_chunk >= chunk_size
+              current_csv&.close
+              sequence += 1
+              rows_in_chunk = 0
+              path = File.join(output_directory, format("%<prefix>s_part_%<num>03d%<ext>s", prefix: file_prefix, num: sequence, ext: ext))
+              raise OutputFileExistsError.new(path) if File.exist?(path) && !overwrite_existing
+
+              chunk_paths << path
+              chunk_row_counts << 0
+              write_mode_headers = headers_present ? row.headers : nil
+              current_csv = ::CSV.open(path, "w", write_headers: write_headers, headers: write_mode_headers, col_sep: col_sep)
+            end
+
+            fields = headers_present ? row.fields : row
+            current_csv << fields
+            rows_in_chunk += 1
+            chunk_row_counts[-1] += 1
+            data_rows += 1
+          end
+
+          {
+            chunk_paths: chunk_paths,
+            chunk_count: chunk_paths.length,
+            data_rows: data_rows,
+            chunk_row_counts: chunk_row_counts
+          }
+        ensure
+          current_csv&.close unless current_csv&.closed?
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/infrastructure/csv/csv_stats_scanner.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Csvtool
+  module Infrastructure
+    module CSV
+      class CsvStatsScanner
+        def initialize(csv: ::CSV)
+          @csv = csv
+        end
+
+        def call(file_path:, col_sep:, headers_present:)
+          data_row_count = 0
+          headers = nil
+          column_count = 0
+          column_stats = []
+
+          # Streaming scan: memory grows with per-column metrics, not row count.
+          @csv.foreach(file_path, headers: headers_present, col_sep: col_sep) do |row|
+            if headers_present
+              headers ||= row.headers
+              column_count = headers.length
+              if column_stats.empty?
+                column_stats = headers.map { |name| { name: name, blank_count: 0, non_blank_count: 0 } }
+              end
+              fields = row.fields
+              fields.fill(nil, fields.length...column_count)
+              fields.each_with_index { |value, index| apply_value(column_stats[index], value) }
+              data_row_count += 1
+            else
+              fields = row.is_a?(::CSV::Row) ? row.fields : row
+              column_count = [column_count, fields.length].max
+              while column_stats.length < column_count
+                column_stats << {
+                  name: "column_#{column_stats.length + 1}",
+                  blank_count: 0,
+                  non_blank_count: 0
+                }
+              end
+              fields.fill(nil, fields.length...column_count)
+              fields.each_with_index { |value, index| apply_value(column_stats[index], value) }
+              data_row_count += 1
+            end
+          end
+
+          {
+            row_count: data_row_count,
+            column_count: column_count,
+            headers: headers,
+            column_stats: column_stats
+          }
+        end
+
+        private
+
+        def apply_value(stats, value)
+          if value.nil? || value.strip.empty?
+            stats[:blank_count] += 1
+          else
+            stats[:non_blank_count] += 1
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/infrastructure/output/csv_split_manifest_writer.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Csvtool
+  module Infrastructure
+    module Output
+      class CsvSplitManifestWriter
+        def call(path:, chunk_paths:, chunk_row_counts:)
+          ::CSV.open(path, "w") do |csv|
+            csv << %w[chunk_index chunk_path row_count]
+            chunk_paths.each_with_index do |chunk_path, index|
+              csv << [index + 1, chunk_path, chunk_row_counts[index]]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/infrastructure/output/csv_stats_file_writer.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Csvtool
+  module Infrastructure
+    module Output
+      class CsvStatsFileWriter
+        def call(path:, data:)
+          ::CSV.open(path, "w") do |csv|
+            csv << %w[metric value]
+            csv << ["row_count", data[:row_count]]
+            csv << ["column_count", data[:column_count]]
+            unless data[:headers].nil? || data[:headers].empty?
+              csv << ["headers", data[:headers].join("|")]
+            end
+            data.fetch(:column_stats, []).each do |stats|
+              csv << ["column.#{stats[:name]}.non_blank", stats[:non_blank_count]]
+              csv << ["column.#{stats[:name]}.blank", stats[:blank_count]]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/errors/presenter.rb

@@ -33,6 +33,10 @@ module Csvtool
             @stdout.puts "Cannot write output file: #{path} (#{error_class})"
           end
 
+          def output_file_exists(path)
+            @stdout.puts "Output file already exists: #{path}"
+          end
+
           def empty_output_path
             @stdout.puts "Output file path cannot be empty."
           end
@@ -53,6 +57,10 @@ module Csvtool
             @stdout.puts "Seed must be an integer."
           end
 
+          def invalid_chunk_size
+            @stdout.puts "Chunk size must be a positive integer."
+          end
+
           def canceled
             @stdout.puts "Canceled."
           end

data/lib/csvtool/interface/cli/menu_loop.rb

@@ -4,7 +4,7 @@ module Csvtool
   module Interface
     module CLI
       class MenuLoop
-        def initialize(stdin:, stdout:, menu_options:, extract_column_action:, extract_rows_action:, randomize_rows_action:, dedupe_action:, parity_action:)
+        def initialize(stdin:, stdout:, menu_options:, extract_column_action:, extract_rows_action:, randomize_rows_action:, dedupe_action:, parity_action:, split_action:, stats_action:)
           @stdin = stdin
           @stdout = stdout
           @menu_options = menu_options
@@ -13,6 +13,8 @@ module Csvtool
           @randomize_rows_action = randomize_rows_action
           @dedupe_action = dedupe_action
           @parity_action = parity_action
+          @split_action = split_action
+          @stats_action = stats_action
         end
 
         def run
@@ -34,9 +36,13 @@ module Csvtool
             when "5"
               @parity_action.call
             when "6"
+              @split_action.call
+            when "7"
+              @stats_action.call
+            when "8"
               return 0
             else
-              @stdout.puts "Please choose 1, 2, 3, 4, 5, or 6."
+              @stdout.puts "Please choose 1, 2, 3, 4, 5, 6, 7, or 8."
             end
           end
         end

data/lib/csvtool/interface/cli/prompts/chunk_size_prompt.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Interface
+    module CLI
+      module Prompts
+        class ChunkSizePrompt
+          def initialize(stdin:, stdout:)
+            @stdin = stdin
+            @stdout = stdout
+          end
+
+          def call
+            @stdout.print "Rows per chunk: "
+            @stdin.gets&.strip.to_s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/prompts/split_manifest_prompt.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Interface
+    module CLI
+      module Prompts
+        class SplitManifestPrompt
+          def initialize(stdin:, stdout:, yes_no_prompt:)
+            @stdin = stdin
+            @stdout = stdout
+            @yes_no_prompt = yes_no_prompt
+          end
+
+          def call(default_path:)
+            write_manifest = @yes_no_prompt.call(
+              label: "Write manifest file? [y/N]: ",
+              default: false
+            )
+            return { write_manifest: false, manifest_path: nil } unless write_manifest
+
+            @stdout.print "Manifest file path [#{default_path}]: "
+            path = @stdin.gets&.strip.to_s
+            path = default_path if path.empty?
+            { write_manifest: true, manifest_path: path }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/prompts/split_output_prompt.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Interface
+    module CLI
+      module Prompts
+        class SplitOutputPrompt
+          def initialize(stdin:, stdout:, yes_no_prompt:)
+            @stdin = stdin
+            @stdout = stdout
+            @yes_no_prompt = yes_no_prompt
+          end
+
+          def call(default_directory:, default_prefix:)
+            @stdout.print "Output directory [#{default_directory}]: "
+            output_directory = @stdin.gets&.strip.to_s
+            output_directory = default_directory if output_directory.empty?
+
+            @stdout.print "Output file prefix [#{default_prefix}]: "
+            file_prefix = @stdin.gets&.strip.to_s
+            file_prefix = default_prefix if file_prefix.empty?
+
+            overwrite_existing = @yes_no_prompt.call(
+              label: "Overwrite existing chunk files? [y/N]: ",
+              default: false
+            )
+
+            {
+              output_directory: output_directory,
+              file_prefix: file_prefix,
+              overwrite_existing: overwrite_existing
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/workflows/builders/csv_split_session_builder.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "csvtool/domain/csv_split_session/split_source"
+require "csvtool/domain/csv_split_session/split_options"
+require "csvtool/domain/csv_split_session/split_session"
+
+module Csvtool
+  module Interface
+    module CLI
+      module Workflows
+        module Builders
+          class CsvSplitSessionBuilder
+            def call(
+              file_path:,
+              col_sep:,
+              headers_present:,
+              chunk_size:,
+              output_directory: nil,
+              file_prefix: nil,
+              overwrite_existing: false,
+              write_manifest: false,
+              manifest_path: nil
+            )
+              source = Domain::CsvSplitSession::SplitSource.new(
+                path: file_path,
+                separator: col_sep,
+                headers_present: headers_present
+              )
+              options = Domain::CsvSplitSession::SplitOptions.new(
+                chunk_size: chunk_size,
+                output_directory: output_directory,
+                file_prefix: file_prefix,
+                overwrite_existing: overwrite_existing,
+                write_manifest: write_manifest,
+                manifest_path: manifest_path
+              )
+              Domain::CsvSplitSession::SplitSession.start(source: source, options: options)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/workflows/builders/csv_stats_session_builder.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "csvtool/domain/csv_stats_session/stats_source"
+require "csvtool/domain/csv_stats_session/stats_options"
+require "csvtool/domain/csv_stats_session/stats_session"
+
+module Csvtool
+  module Interface
+    module CLI
+      module Workflows
+        module Builders
+          class CsvStatsSessionBuilder
+            def call(file_path:, col_sep:, headers_present:, destination:)
+              source = Domain::CsvStatsSession::StatsSource.new(
+                path: file_path,
+                separator: col_sep,
+                headers_present: headers_present
+              )
+              options = Domain::CsvStatsSession::StatsOptions.new
+              session = Domain::CsvStatsSession::StatsSession.start(source: source, options: options)
+              session.with_output_destination(destination)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/workflows/presenters/csv_split_presenter.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Interface
+    module CLI
+      module Workflows
+        module Presenters
+          class CsvSplitPresenter
+            def initialize(stdout:)
+              @stdout = stdout
+            end
+
+            def print_summary(data)
+              @stdout.puts "Split complete."
+              @stdout.puts "Chunk size: #{data[:chunk_size]}"
+              @stdout.puts "Data rows: #{data[:data_rows]}"
+              @stdout.puts "Chunks written: #{data[:chunk_count]}"
+              @stdout.puts "Manifest: #{data[:manifest_path]}" if data[:manifest_path]
+              data[:chunk_paths].each { |path| @stdout.puts path }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/csvtool/interface/cli/workflows/presenters/csv_stats_presenter.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Csvtool
+  module Interface
+    module CLI
+      module Workflows
+        module Presenters
+          class CsvStatsPresenter
+            def initialize(stdout:)
+              @stdout = stdout
+            end
+
+            def print_summary(data)
+              @stdout.puts "CSV Stats Summary"
+              @stdout.puts "Rows: #{data[:row_count]}"
+              @stdout.puts "Columns: #{data[:column_count]}"
+              @stdout.puts "Headers: #{data[:headers].join(', ')}" unless data[:headers].nil? || data[:headers].empty?
+              return if data[:column_stats].nil? || data[:column_stats].empty?
+
+              @stdout.puts "Column completeness:"
+              data[:column_stats].each do |stats|
+                @stdout.puts "  #{stats[:name]}: non_blank=#{stats[:non_blank_count]} blank=#{stats[:blank_count]}"
+              end
+            end
+
+            def print_file_written(path)
+              @stdout.puts "Wrote output to #{path}"
+            end
+          end
+        end
+      end
+    end
+  end
+end