data_collector 0.35.0 → 0.36.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b9d99a5bf660ca9feb5bea5dd13f12c2539dcb224d6689b7a1181af32d0ab20
-  data.tar.gz: 84fa315a8e2fa932caf84cad75d9c86c643c8d3b1faa389b383125de23022630
+  metadata.gz: 649b618a7676b2153451b2bc3053726dd332ccc95a58ce0fe7115f4c9cc5a7eb
+  data.tar.gz: 625ca0e9da4e6b986d7ead973c80fc04a87a5539f0f7263efc672ee0f3ef7eb1
 SHA512:
-  metadata.gz: 82cb1c183a2d7a8ea950a181f2ae9571720533b6defd378bc0cab91fc84d876c75fc71029cd1f46c993c0b004eee7ecf32635bff808cda4419b3236ea7206c42
-  data.tar.gz: 3bf761a214120ed1d12aad599e8137ce9a9b2028c757a7e59117d0ac3300e307237160cfb359032d8506db8efc37aee3669604bba0cbac59b60b51e064fc79e8
+  metadata.gz: 2372b50b9fc9e6f5a17575c1755e169f0ddbaef44ed96847d8f5c87b28a3e30e46276cf47f6e506fe843a2293190b54fd556978ea175f937693abd4f83a637dd
+  data.tar.gz: 7a417a9ee3a70140a3601d20613159222f6c2d6d0d7feaa99c447c7e31306f14c6f08c50543cb8fda85315b5fe62abde202e2b3bda51ac421f6a517f7b6be740

data/README.md

@@ -403,6 +403,13 @@ Should give as output
 </data>
 ```
 
+You can provide options to input.from_uri for better reading CSV formats these 
+are the same the Ruby [CSV](https://docs.ruby-lang.org/en/master/CSV.html#class-CSV-label-Options) class
+
+Loading a CSV file with **;** as the row seperator
+```ruby
+i = input.from_uri('https://support.staffbase.com/hc/en-us/article_attachments/360009197031/username.csv', col_sep: ';')
+```
 
 ## Installation
 

data/lib/data_collector/input.rb

@@ -161,7 +161,7 @@ module DataCollector
           when 'application/atom+xml'
             data = xml_to_hash(data, options)
           when 'text/csv'
-            data = csv_to_hash(data)
+            data = csv_to_hash(data, options)
           when 'application/xml'
             data = xml_to_hash(data, options)
           when 'text/xml'
@@ -231,7 +231,7 @@ module DataCollector
             end #entry
           end #tar
         when '.csv'
-          data = csv_to_hash(data)
+          data = csv_to_hash(data, options)
         else
           raise "Do not know how to process #{uri.to_s}"
         end
@@ -264,8 +264,15 @@ module DataCollector
       nori.parse(data)
     end
 
-    def csv_to_hash(data)
-      csv = CSV.parse(data, headers: true, header_converters: [:downcase, :symbol])
+    def csv_to_hash(data, options = {})
+      csv_option_keys = options.keys & CSV::DEFAULT_OPTIONS.keys
+      all_cvs_options = {headers: true, header_converters: [:downcase, :symbol]}
+
+      csv_option_keys.each do |k|
+        all_cvs_options[k] = options[k]
+      end
+
+      csv = CSV.parse(data, **all_cvs_options)
 
       csv.collect do |record|
         record.to_hash

data/lib/data_collector/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module DataCollector
-  VERSION = "0.35.0"
+  VERSION = "0.36.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_collector
 version: !ruby/object:Gem::Version
-  version: 0.35.0
+  version: 0.36.0
 platform: ruby
 authors:
 - Mehmet Celik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-11-13 00:00:00.000000000 Z
+date: 2023-11-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport