metacrunch-file 1.4.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1a64cc33b8badb7d4e6b7c29267501db505edbb081ed55be3c4bdf269109eac8
-  data.tar.gz: 0bb18d835fb8e812ff04aded3b8bfc3d90f8daae606704c49d2dbfe502cdaafd
+  metadata.gz: 6fbb7eafc182c80e2bbb2a1c63b57e02824df6739a0f123c32d609997ceaaf9c
+  data.tar.gz: ee41b433ea1ea30109074c2043213ff928baa2fd516c735545392294dc3fa800
 SHA512:
-  metadata.gz: a28ff6fcb40f286673011ee27de86a15d579bdc213b379eb77781eddbd6efa5613e3d68bd16e1d016d61f9d786b143c35e2b276961affbad21d71707060c65a4
-  data.tar.gz: e58ec6254b11f956c4a45a0ef68477cee7b9bb2e79ce5fd283f5a3e0450e0e4b49c5c53796f7e6234ad9742d46fd42e19fed6bd2be231847b8de6a82cb188d57
+  metadata.gz: 7e26d47271466d6df0ec80c80903741895529a037e1bf1d8b03d751effba20a7f292cea7cc704856b7d5cb4bc1eabe2f0f493bde32eca07e22595bb43c63b495
+  data.tar.gz: f9dadce5be892b1f3fec41e3490b06d7b5826964132ff9b2e1c3077f998823e51d8347a426b75c712e434f3ae6a3f92fd29c0db4a66ca3323414c2cdfe3e16e8

data/Readme.md

@@ -6,7 +6,7 @@ metacrunch-file
 [![Test Coverage](https://codeclimate.com/github/ubpb/metacrunch-file/badges/coverage.svg)](https://codeclimate.com/github/ubpb/metacrunch-file/coverage)
 [![CircleCI](https://circleci.com/gh/ubpb/metacrunch-file.svg?style=svg)](https://circleci.com/gh/ubpb/metacrunch-file)
 
-This is the official file package for the [metacrunch ETL toolkit](https://github.com/ubpb/metacrunch). 
+This is the official file package for the [metacrunch ETL toolkit](https://github.com/ubpb/metacrunch).
 
 *Note: For working examples on how to use this package check out our [demo repository](https://github.com/ubpb/metacrunch-demo).*
 
@@ -17,7 +17,7 @@ Installation
 Include the gem in your `Gemfile`
 
 ```ruby
-gem "metacrunch-file", "~> 1.4.0"
+gem "metacrunch-file", "~> 1.5.0"
 ```
 
 and run `$ bundle install` to install it.
@@ -52,7 +52,7 @@ source Metacrunch::File::FileSource.new(["my-data.xml", "my-other-data.xml", "..
 
 NONE.
 
-The source yields objects of type `Metacrunch::File::Entry` for every file it reads. 
+The source yields objects of type `Metacrunch::File::Entry` for every file it reads.
 
 ```ruby
 # my_job.metacrunch
@@ -82,21 +82,39 @@ destination Metacrunch::File::FileDestination.new("/tmp/my-data.txt" [, OPTIONS]
 
 ## `Metacrunch::File::CSVSource`
 
-This class provides a metacrunch `source` for reading CSV files. It is a simple wrapper around [smarter_csv](https://github.com/tilo/smarter_csv) gem. 
+This class provides a metacrunch `source` for reading CSV files. It is a simple wrapper around [smarter_csv](https://github.com/tilo/smarter_csv) gem.
 
 ```ruby
 # my_job.metacrunch
 
-source Metacrunch::File::CSVSource.new("my.csv" [, OPTIONS])
+source Metacrunch::File::CSVSource.new(
+  "source.csv" # filename
+  [, OPTIONS]) # options
 ```
 
 **Options**
 
-* `headers`: Whether or not the file contains headers as the first line. Important if the file does not contain headers, otherwise you would lose the first line of data. Defaults to `true`.
-* `col_sep`: Column separator. Defaults to `,`.
-* `row_sep`: Row separator or record separator. Defaults to `\n`.
-* `quote_char`: Quotation character. Defaults to `"`.
-* `file_encoding`: Set the file encoding. Defaults to `utf-8`.
+Using the `options` argument you can pass any [CSV reading option supported by smarter_csv](https://github.com/tilo/smarter_csv/blob/main/docs/options.md#csv-reading) using the key `csv_options`.
+
+* `csv_options`: Hash with any option supported by smarter_csv for CVS reading. Our defaults are `headers_in_file: true`, `col_sep: ","`, `row_sep: "\n"`, `quote_char: '"'`, `file_encoding: "utf-8"`
+
+## `Metacrunch::File::CSVDestination`
+
+This class provides a metacrunch `desination` for writing CSV files. Like the `CSVSource` this uses [smarter_csv](https://github.com/tilo/smarter_csv) under the hood.
+
+```ruby
+# my_job.metacrunch
+
+destination Metacrunch::File::CSVDestination.new(
+  "result.csv" # filename
+  [, OPTIONS]  # options
+)
+```
+
+**Options**
+
+* `override_existing_file`: Overrides an existing file if set to `true`. If set to `false` an error is raised if the file already exists. Defaults to `false`.
+* `csv_options`: Set options for CSV generation as `col_sep`. Full list is [here](https://ruby-doc.org/stdlib/libdoc/csv/rdoc/CSV.html#class-CSV-label-Options).
 
 ## `Metacrunch::File::XLSXDestination`
 

data/lib/metacrunch/file/csv_destination.rb

@@ -0,0 +1,32 @@
+require "metacrunch/file"
+require "smarter_csv"
+
+module Metacrunch
+  class File::CSVDestination
+
+    DEFAULT_OPTIONS = {
+      override_existing_file: false,
+      csv_options: {}
+    }
+
+    def initialize(filename, options = {})
+      @filename = ::File.expand_path(filename)
+      @options  = DEFAULT_OPTIONS.deep_merge(options)
+
+      if ::File.exist?(@filename) && @options[:override_existing_file] == false
+        raise "File `#{@filename}` exists but `override_existing_file` option was set to `false`"
+      end
+
+      @csv_writer = SmarterCSV::Writer.new(@filename, @options[:csv_options])
+    end
+
+    def write(data)
+      @csv_writer << data
+    end
+
+    def close
+      @csv_writer.finalize if @csv_writer
+    end
+
+  end
+end

data/lib/metacrunch/file/csv_source.rb

@@ -5,11 +5,13 @@ module Metacrunch
   class File::CSVSource
 
     DEFAULT_OPTIONS = {
-      headers: true,
-      col_sep: ",",
-      row_sep: "\n",
-      quote_char: '"',
-      file_encoding: "utf-8"
+      csv_options: {
+        headers_in_file: true,
+        col_sep: ",",
+        row_sep: "\n",
+        quote_char: '"',
+        file_encoding: "utf-8"
+      }
     }
 
     def initialize(csv_filename, options = {})
@@ -20,13 +22,7 @@ module Metacrunch
     def each(&block)
       return enum_for(__method__) unless block_given?
 
-      SmarterCSV.process(@filename, {
-        headers_in_file: @options[:headers],
-        col_sep: @options[:col_sep],
-        row_sep: @options[:row_sep],
-        quote_char: @options[:quote_char],
-        file_encoding: @options[:file_encoding]
-      }) do |line|
+      SmarterCSV.process(@filename, @options[:csv_options]) do |line|
         yield line
       end
     end

data/lib/metacrunch/file/file_destination.rb

@@ -11,7 +11,7 @@ module Metacrunch
       @filename = ::File.expand_path(filename)
       @options = DEFAULT_OPTIONS.deep_merge(options)
 
-      if ::File.exists?(@filename) && @options[:override_existing_file] == false
+      if ::File.exist?(@filename) && @options[:override_existing_file] == false
         raise "File `#{@filename}` exists but `override_existing_file` option was set to `false`"
       end
 

data/lib/metacrunch/file/version.rb

@@ -1,5 +1,5 @@
 module Metacrunch
   module File
-    VERSION = "1.4.0"
+    VERSION = "2.0.0"
   end
 end

data/lib/metacrunch/file.rb

@@ -10,5 +10,6 @@ module Metacrunch
     require_relative "file/file_destination"
     require_relative "file/destination"
     require_relative "file/xlsx_destination"
+    require_relative "file/csv_destination"
   end
 end

data/metacrunch-file.gemspec

@@ -16,6 +16,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "activesupport", ">= 0"
-  spec.add_dependency "caxlsx", "~> 3.0"
+  spec.add_dependency "caxlsx", "~> 4.0"
   spec.add_dependency "smarter_csv", "~> 1.2"
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: metacrunch-file
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 2.0.0
 platform: ruby
 authors:
 - René Sprotte
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-13 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -30,14 +29,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: smarter_csv
   requirement: !ruby/object:Gem::Requirement
@@ -52,8 +51,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
-description:
-email:
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -67,6 +64,7 @@ files:
 - Readme.md
 - bin/console
 - lib/metacrunch/file.rb
+- lib/metacrunch/file/csv_destination.rb
 - lib/metacrunch/file/csv_source.rb
 - lib/metacrunch/file/destination.rb
 - lib/metacrunch/file/entry.rb
@@ -80,7 +78,6 @@ homepage: http://github.com/ubpb/metacrunch-file
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -95,8 +92,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key:
+rubygems_version: 3.6.9
 specification_version: 4
 summary: File package for the metacrunch ETL toolkit.
 test_files: []