large_csv_reader 1.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 12f98dccfdfdc9e1c4b9cfee61c45fabc328338707d3ea39bda7ad98935d9daf
+  data.tar.gz: 40ca5ad5f66d00b6ab02c7236c03cdb5d53889c32687415cf3e321e74a2a6144
+SHA512:
+  metadata.gz: d587014c2ed754c4c82b557066bef98b41411a996c8e365112dab428559749eb2080ae15f5cb405738c915d38e08bf3b362821a33d7cc0847748b4a23742b126
+  data.tar.gz: 77c821fde2268965a976f88c4ecb99e55b7466f4626120cf65afddab81585d5654cb7b4e97fd1973c2008fe9c57f30353b24ee57d4b711ede74e00180ed756e3

data/README.md

@@ -0,0 +1,32 @@
+# Large CSV Reader GEM
+
+This **gem** is created to help with a known issue, processing large **csv files** using **ruby,** without breaking our **RAM.** I used lazy enumeration to achieve this, that allows the methods to execute actions on one line at a time instead of loading millions of lines in memory at once.
+
+I consider this **gem** in **Beta state**, proper testing suite is still missing and there are some improvements or extensions that could be useful to have in a full version of it.
+
+
+# Installation
+`gem install large_csv_reader`
+- In your files:
+	`require 'large_csv_reader'`
+
+# Usage
+
+## General Methods
+|Function|Description  |
+|--|--|
+|`reader = LargeCsvReader.new`| creates a new instance of the reader|
+|`reader.generate_csv(fileName, columnNames)`|creates a new csv file with the name and header names passed as parameters|
+|`reader.append_to_csv(filename, rows=1000000,rowStructure)`|Add lines to the csv, this lines are generated with the rowStructure array parameter. If rows parameters is not present by default it will load 1 million lines to the file.|
+|`reader.massive_read_in_csv_data (file_name)`|lazy load of each csv row into a list|
+|`reader.massive_csv_builder(filename, column_names,rowMult="1")`|create a csv with millions of lines, the value of rowMult represents how many millions lines the file will have|
+|`reader.row_generator(structure)`|generate rows on demand using enumeration|
+
+## Specific Methods
+The **rest** of the methods are considerations to solve a **test problem** with **book data** "**Date**", "**ISBN**", "**Price**"
+
+ - `massive_total_value_in_stock(csv_file_name)`
+ - `massive_number_of_each_isbn(csv_file_name)`
+ - `append_book_to_csv(filename,rows=1000000)`
+ - `book_generator`
+    

data/large_csv_reader.gemspec

@@ -0,0 +1,12 @@
+Gem::Specification.new do |s|
+    s.name        = 'large_csv_reader'
+    s.version     = '1.0.1'
+    s.summary     = "Large CSV builder and proccesor"
+    s.description = "Large CSV builder and proccesor, it can help with creation of large csv files, reading those files and make operation over the csv data"
+    s.authors     = ["Arturo Castro"]
+    s.email       = 'agcastro1994@gmail.com'
+    s.files       = ["lib/large_csv_reader.rb", "lib/large_csv_reader/book_in_stock.rb", "large_csv_reader.gemspec", "README.md"]    
+    s.license       = ''
+    s.add_dependency 'csv', '~> 1.0'
+    s.add_dependency 'date', '~> 1.0'
+  end

data/lib/large_csv_reader/book_in_stock.rb

@@ -0,0 +1,8 @@
+class BookInStock
+    attr_reader :isbn
+    attr_accessor :price
+    def initialize(isbn, price)
+        @isbn = isbn
+        @price = Float(price)
+    end
+end

data/lib/large_csv_reader.rb

@@ -0,0 +1,137 @@
+require 'csv'
+require 'date'
+require 'large_csv_reader/book_in_stock'
+
+class LargeCsvReader
+    attr_accessor :books_in_stock
+    def initialize
+        @books_in_stock = []
+    end
+
+    def massive_csv_builder(filename,column_names,rowStructure,rowMultiplicator="1")
+        generate_csv(filename,column_names)
+        millions = rowMultiplicator.to_i
+        millions.times do 
+            append_to_csv(filename, rowStructure)
+        end
+    end
+
+    def massive_book_csv_builder(filename,column_names,rowMultiplicator="1")
+        generate_csv(filename,column_names)
+        millions = rowMultiplicator.to_i
+        millions.times do 
+            append_book_to_csv(filename)
+        end
+    end
+
+    def generate_csv(filename,column_names)       
+        CSV.open(filename, "ab") do |csv|
+            csv << column_names           
+        end
+    end
+
+    def append_book_to_csv(filename,rows=1000000)
+        CSV.open(filename, "ab") do |csv|          
+            self.book_generator.lazy.select {|book| csv << [Date.new(2001,2,3), book.isbn, book.price] }.first(rows)        
+        end
+    end
+
+    def append_to_csv(filename,rows=1000000,rowStructure)
+        CSV.open(filename, "ab") do |csv|          
+            self.row_generator(rowStructure).lazy.select {|row| csv << row }.first(rows)        
+        end
+    end
+
+    def massive_read_in_csv_data(csv_file_name)
+        CSV.foreach(csv_file_name, headers: true).each_slice(1).each {|row| @books_in_stock << BookInStock.new(row[0][1], row[0][2])}
+    end
+
+    def massive_total_value_in_stock(csv_file_name)
+        CSV.foreach(csv_file_name, headers: true).each_slice(1).each.inject(0) {|sum, row| sum + row[0][2].to_f }
+    end
+
+    def massive_number_of_each_isbn(csv_file_name)
+        counter_hash = {}
+        CSV.foreach(csv_file_name, headers: true).each_slice(1).each {|row| number_of_each_isbn(counter_hash, row) }
+        return counter_hash
+    end
+
+    def number_of_each_isbn(counter_hash, bookInfo)        
+        isbn = bookInfo[0][1]
+        if counter_hash.has_key?(isbn)
+            counter_hash[isbn] = counter_hash[isbn]+1
+        else
+            counter_hash[isbn] = 1
+        end        
+        counter_hash
+    end
+
+    def book_generator
+        Enumerator.new do |caller|  
+            testBook = BookInStock.new("978-1-9343561-0-4", 20.05)                    
+            loop do
+                caller.yield testBook                
+            end
+        end
+    end
+
+    def row_generator(structure)
+        Enumerator.new do |caller|  
+            row = structure                    
+            loop do
+                caller.yield row                
+            end
+        end
+    end
+
+    # def add_new_book(book, filename)
+    #     CSV.open(filename, "ab") do |csv|           
+    #         csv << [Date.new(2001,2,3), book.isbn, book.price]
+    #     end
+    # end
+
+    # def total_value_in_stock
+    #     sum = 0.0
+    #     @books_in_stock.each do |book|
+    #         sum += book.price
+    #     end
+    #     sum
+    # end
+
+        # def read_in_csv_data(csv_file_name)
+    #     CSV.foreach(csv_file_name, headers: true) do |row|
+    #         @books_in_stock << BookInStock.new(
+    #             row["ISBN"],
+    #             row["Price"]
+    #         )
+    #     end
+    # end
+
+    # def generate_csv(filename,column_names, rows)
+    #     records = self.book_generator.first(rows)
+        
+    #     CSV.open(filename, "ab") do |csv|
+    #         csv << column_names
+    #         10.times do 
+    #             records.each do |book|
+    #                 csv << [Date.new(2001,2,3), book.isbn, book.price]
+    #             end
+    #         end
+    #     end
+    # end
+
+    # def number_of_each_isbn
+        
+    #     counter_hash = {}
+
+    #     @books_in_stock.each do |book|
+    #         isbn = book.isbn
+    #         if counter_hash.has_key?(isbn)
+    #             counter_hash[isbn] = counter_hash[isbn]+1
+    #         else
+    #             counter_hash[isbn] = 1
+    #         end
+    #     end
+    #     counter_hash
+    # end
+end

metadata

@@ -0,0 +1,75 @@
+--- !ruby/object:Gem::Specification
+name: large_csv_reader
+version: !ruby/object:Gem::Version
+  version: 1.0.1
+platform: ruby
+authors:
+- Arturo Castro
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2022-05-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: date
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: Large CSV builder and proccesor, it can help with creation of large csv
+  files, reading those files and make operation over the csv data
+email: agcastro1994@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- large_csv_reader.gemspec
+- lib/large_csv_reader.rb
+- lib/large_csv_reader/book_in_stock.rb
+homepage: 
+licenses:
+- ''
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.14
+signing_key: 
+specification_version: 4
+summary: Large CSV builder and proccesor
+test_files: []