eco-helpers 2.7.12 → 2.7.13

data/lib/eco/csv/split.rb

@@ -0,0 +1,114 @@
+module Eco
+  class CSV
+    class Split
+      include Eco::Language::AuxiliarLogger
+
+      attr_reader :filename
+
+      def initialize(filename, max_rows:, **kargs)
+        raise ArgumentError, "File '#{filename}' does not exist" unless ::File.exist?(filename)
+        @filename = filename
+        @max_rows = max_rows
+        @params   = kargs
+        init
+      end
+
+      # @yield [idx, file] a block to spot the filename
+      # @yieldparam idx [Integer] the number of the file
+      # @yieldparam file [String] the default name of the file
+      # @yieldreturn [String] the filename of the file `idx`.
+      #   - If `nil` it will create its own filename convention
+      # @return [Array<String>] names of the generated files
+      def call(&block)
+        stream.for_each do |row, ridx|
+          copy_row(row, ridx, &block)
+        end
+        out_files
+      ensure
+        puts "Close at row #{row_idx}"
+        @csv&.close
+      end
+
+      private
+
+      attr_reader :params
+      attr_reader :idx, :max_rows
+      attr_reader :headers, :row_idx
+
+      attr_accessor :exception
+
+      def copy_row(row, ridx, &block)
+        @headers ||= row.headers
+        @row_idx   = ridx
+        current_csv(ridx, &block) << row.fields
+      end
+
+      def current_csv(ridx)
+        if split?(ridx) || @csv.nil?
+          puts "Split at row #{row_idx}"
+          @csv&.close
+          out_filename = generate_name(nidx = next_idx)
+          out_filename = yield(nidx, out_filename) if block_given?
+          @csv       = ::CSV.open(out_filename, "w")
+          @csv      << headers
+          out_files << out_filename
+        end
+        @csv
+      end
+
+      def split?(ridx)
+        ((ridx + 1) % max_rows).zero?
+      end
+
+      def next_idx
+        idx.tap { @idx += 1 }
+      end
+
+      def init
+        @idx ||= 0 # rubocop:disable Naming/MemoizedInstanceVariableName
+      end
+
+      def stream
+        @stream ||= Eco::CSV::Stream.new(filename, **params)
+      end
+
+      def generate_name(fidx)
+        File.join(input_dir, "#{input_name}_#{file_number(fidx)}#{input_ext}")
+      end
+
+      def file_number(num)
+        "#{zeroed}#{num}"[-5..]
+      end
+
+      def zeroed
+        "0" * 5
+      end
+
+      def out_files
+        @out_files ||= []
+      end
+
+      def input_name
+        @input_name ||= File.basename(input_basename, input_ext)
+      end
+
+      def input_ext
+        @input_ext ||= input_basename.split('.')[1..].join('.').then do |name|
+          ".#{name}"
+        end
+      end
+
+      def input_basename
+        @input_basename ||= File.basename(input_full_filename)
+      end
+
+      def input_dir
+        @input_dir = File.dirname(input_full_filename)
+      end
+
+      def input_full_filename
+        @input_full_filename ||= File.expand_path(filename)
+      end
+    end
+  end
+end

data/lib/eco/csv/stream.rb

@@ -0,0 +1,66 @@
+module Eco
+  class CSV
+    class Stream
+      include Eco::Language::AuxiliarLogger
+
+      attr_reader :filename
+
+      def initialize(filename, **kargs)
+        raise ArgumentError, "File '#{filename}' does not exist" unless ::File.exist?(filename)
+        @filename = filename
+        @params   = {
+          headers:     true,
+          skip_blanks: true
+        }.merge(kargs)
+        init
+      end
+
+      def for_each(start_at_idx: 0)
+        raise ArgumentError, 'Expecting block, but not given.' unless block_given?
+
+        move_to_idx(start_at_idx)
+
+        yield(row, next_idx) while (self.row = csv.shift)
+      rescue StandardError => err
+        self.exception = err
+        raise
+      ensure
+        (fd.close; @fd = nil) if fd.is_a?(::File) # rubocop:disable Style/Semicolon
+        if exception
+          # Give some feedback if it crashes
+          msg  = []
+          msg << "Last row IDX: #{idx}"
+          msg << "Last row content: #{row.to_h.pretty_inspect}"
+          puts msg
+          log(:debug) { msg.join("\n") }
+        end
+      end
+
+      def move_to_idx(start_at_idx)
+        next_idx while (idx < start_at_idx) && (self.row = csv.shift)
+      end
+
+      private
+
+      attr_reader :params
+      attr_reader :idx, :fd
+
+      attr_accessor :row, :exception
+
+      def next_idx
+        idx.tap { @idx += 1 }
+      end
+
+      # see https://dalibornasevic.com/posts/68-processing-large-csv-files-with-ruby
+      def csv
+        return @csv if instance_variable_defined?(:@csv)
+        @fd  = ::File.open(filename, 'r')
+        @csv = Eco::CSV.new(fd, **params)
+      end
+
+      def init
+        @idx ||= 0 # rubocop:disable Naming/MemoizedInstanceVariableName
+      end
+    end
+  end
+end

data/lib/eco/csv.rb

@@ -17,8 +17,22 @@ module Eco
         end
         parse(get_file_content(file, **params), **kargs)
       end
+
+      # @yield [idx, file] a block to spot the filename
+      # @yieldparam idx [Integer] the number of the file
+      # @yieldparam file [String] the default name of the file
+      # @yieldreturn [String] the filename of the file `idx`.
+      #   - If `nil` it will create its own filename convention
+      # @param filename [String] the orignal file
+      # @param max_rows [Integer] number of rows per file
+      # @see Eco::CSV::Split#call
+      def split(filename, max_rows:, **kargs, &block)
+        Eco::CSV::Split.new(filename, max_rows: max_rows, **kargs).call(&block)
+      end
     end
   end
 end
 
 require_relative 'csv/table'
+require_relative 'csv/stream'
+require_relative 'csv/split'

data/lib/eco/language/basic_logger.rb

@@ -19,12 +19,12 @@ module Eco
       attr_writer :timestamp
       attr_reader :level
 
-      forward *LOG_LEVELS, *METHODS
+      forward(*LOG_LEVELS, *METHODS)
 
       def initialize(level: ::Logger::INFO, timestamp: false)
         @level             = level
         self.timestamp     = timestamp
-        loggers[:console]  = ::Logger.new(STDOUT).tap do |logger|
+        loggers[:console]  = ::Logger.new($stdout).tap do |logger|
           logger.formatter = format_proc(console: true)
           logger.level     = level
         end
@@ -41,7 +41,7 @@ module Eco
       private
 
       def forward(meth, *args, &block)
-        loggers.each do |_key, logger|
+        loggers.each_value do |logger|
           logger.send(meth, *args, &block)
         end
       end
@@ -52,6 +52,7 @@ module Eco
 
       def console_timestamp(datetime)
         return nil unless timestamp?
+
         timestamp(datetime)
       end
 
@@ -62,10 +63,10 @@ module Eco
       end
 
       def format_proc(console: true, &block)
-        proc do |severity, datetime, progname, msg|
-          str_stamp     = console ? console_timestamp(datetime) : timestamp(datetime)
+        proc do |severity, datetime, _progname, msg|
+          str_stamp = console ? console_timestamp(datetime) : timestamp(datetime)
           "#{severity.to_s[0]}: #{str_stamp}#{msg}\n".tap do |formatted_msg|
-            block.call(severity, datetime, msg, formatted_msg) if block
+            block&.call(severity, datetime, msg, formatted_msg)
           end
         end
       end

data/lib/eco/version.rb

@@ -1,3 +1,3 @@
 module Eco
-  VERSION = '2.7.12'.freeze
+  VERSION = '2.7.13'.freeze
 end