metacrunch-file 1.2.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d530edbf89a1317841d79de98e9e88f6c865c78bb1b26b8f838581308f4f2ddb
-  data.tar.gz: 1cf9fe7d0dfcc65a2e0bc37ecb146d376005bffd4e70aae8822eb92ab50bcce1
+  metadata.gz: 9980829422c838552b9334134e52932f3d70d132859965baa70a93f184ffefaf
+  data.tar.gz: 81b3b88c8725eb30469eb1dd40a7f9507865e66a998b76ff2ace5691e424b684
 SHA512:
-  metadata.gz: 86a35a6bc99eaa6ac6aaca784cf7dfeb626f1c7378c03fa9de5a17f250c51a05fc8c7febda42f34c90302cb6e05f39da08f506b35645095feac585466a1830a8
-  data.tar.gz: 2d96c0cf6eb38eb21ed193a8da58ed06cbd837e8a2d9c799fb96ac5d7cc7871564972dd7d3f2e4b70eb42213730073f052dce01e14ede1be12d71525734b3275
+  metadata.gz: '090856c4f3cef0b0bc2bedaa2a5fd87ef73005b4678e2bbe5a938475438701b1ed74307dc502bfd5f52614589ed6ef1f55667037d0c1599ecd8ae8c1fb68d8dc'
+  data.tar.gz: b940c2ea0555e018303de286a028e975d1cd628cd527d65ca702afb97d7bbac40cc31cad67b27871b3737891b65f016ccff31b960ba761af0cb4157cb033195e

data/Readme.md

@@ -6,7 +6,10 @@ metacrunch-file
 [![Test Coverage](https://codeclimate.com/github/ubpb/metacrunch-file/badges/coverage.svg)](https://codeclimate.com/github/ubpb/metacrunch-file/coverage)
 [![CircleCI](https://circleci.com/gh/ubpb/metacrunch-file.svg?style=svg)](https://circleci.com/gh/ubpb/metacrunch-file)
 
-This is the official file package for the [metacrunch ETL toolkit](https://github.com/ubpb/metacrunch).
+This is the official file package for the [metacrunch ETL toolkit](https://github.com/ubpb/metacrunch). 
+
+*Note: For working examples on how to use this package check out our [demo repository](https://github.com/ubpb/metacrunch-demo).*
+
 
 Installation
 ------------
@@ -14,7 +17,7 @@ Installation
 Include the gem in your `Gemfile`
 
 ```ruby
-gem "metacrunch-file", "~> 1.2.0"
+gem "metacrunch-file", "~> 1.3.0"
 ```
 
 and run `$ bundle install` to install it.
@@ -29,24 +32,20 @@ $ gem install metacrunch-file
 Usage
 -----
 
-*Note: For working examples on how to use this package check out our [demo repository](https://github.com/ubpb/metacrunch-demo).*
-
-### `Metacrunch::File::Source`
+## `Metacrunch::File::FileSource`
 
 This class provides a metacrunch `source` implementation that can be used to read data from files in the file system into a metacrunch job. The class can be used to read regular files, compressed files (gzip), tar archives and compressed tar archives (gzip).
 
-You can access non-option arguments from the command line using the `ARGV` constant.
-
 ```ruby
 # my_job.metacrunch
 
 # If you call this example like so
 #   $ metacrunch my_job.metacrunch *.xml
 # ARGV will contain all the XML files in the current directory.
-source Metacrunch::File::Source.new(ARGV)
+source Metacrunch::File::FileSource.new(ARGV)
 
 # ... or you can set the filenames directly
-source Metacrunch::File::Source.new(["my-data.xml", "my-other-data.xml", "..."])
+source Metacrunch::File::FileSource.new(["my-data.xml", "my-other-data.xml", "..."])
 ```
 
 **Options**
@@ -67,22 +66,33 @@ transformation ->(file_entry) do
 end
 ```
 
-### `Metacrunch::File::Destination`
+## `Metacrunch::File::FileDestination`
 
 This class provides a metacrunch `destination` to write data to a file. Every data that gets passed to the destination is appended to the given file. If the data is an `Array` every element of that array is appended to the file. Non existing files will be created automatically.
 
 ```ruby
 # my_job.metacrunch
 
-source Metacrunch::File::Destination.new("/tmp/my-data.txt" [, OPTIONS])
+destination Metacrunch::File::FileDestination.new("/tmp/my-data.txt" [, OPTIONS])
 ```
 
 **Options**
 
 * `override_existing_file`: Overrides an existing file if set to `true`. If set to `false` an error is raised if the file already exists. Defaults to `false`.
-* 
 
-### `Metacrunch::File::XLSXDestination`
+## `Metacrunch::File::CSVSource`
+
+This class provides a metacrunch `source` for reading CSV files. It is a simple wrapper around [smarter_csv](https://github.com/tilo/smarter_csv) gem. 
+
+**Options**
+
+* `headers`: Whether or not the file contains headers as the first line. Important if the file does not contain headers, otherwise you would lose the first line of data. Defaults to `true`.
+* `col_sep`: Column separator. Defaults to `,`.
+* `row_sep`: Row separator or record separator. Defaults to `\n`.
+* `quote_char`: Quotation character. Defaults to `"`.
+* `file_encoding`: Set the file encoding. Defaults to `utf-8`.
+
+## `Metacrunch::File::XLSXDestination`
 
 This class provides a metacrunch `destination` implementation to create simple Excel (xlsx) files.
 
@@ -95,7 +105,7 @@ transformation ->(data) do
   [data["foo"], data["bar"], ...]
 end
 
-source Metacrunch::File::XLSXDestination.new(
+destination Metacrunch::File::XLSXDestination.new(
     "/tmp/my-data.xlsx",           # filename
     ["Column 1", "Column 2", ...], # header columns
     OPTIONS

data/lib/metacrunch/file.rb

@@ -4,7 +4,10 @@ require "active_support/core_ext"
 module Metacrunch
   module File
     require_relative "file/entry"
+    require_relative "file/csv_source"
+    require_relative "file/file_source"
     require_relative "file/source"
+    require_relative "file/file_destination"
     require_relative "file/destination"
     require_relative "file/xlsx_destination"
   end

data/lib/metacrunch/file/csv_source.rb

@@ -0,0 +1,35 @@
+require "metacrunch/file"
+require "smarter_csv"
+
+module Metacrunch
+  class File::CSVSource
+
+    DEFAULT_OPTIONS = {
+      headers: true,
+      col_sep: ",",
+      row_sep: "\n",
+      quote_char: '"',
+      file_encoding: "utf-8"
+    }
+
+    def initialize(csv_filename, options = {})
+      @filename = csv_filename
+      @options = DEFAULT_OPTIONS.merge(options)
+    end
+
+    def each(&block)
+      return enum_for(__method__) unless block_given?
+
+      SmarterCSV.process(@filename, {
+        headers_in_file: @options[:headers],
+        col_sep: @options[:col_sep],
+        row_sep: @options[:row_sep],
+        quote_char: @options[:quote_char],
+        file_encoding: @options[:file_encoding]
+      }) do |line|
+        yield line
+      end
+    end
+
+  end
+end

data/lib/metacrunch/file/destination.rb

@@ -1,35 +1,11 @@
 require "metacrunch/file"
 
 module Metacrunch
-  class File::Destination
-
-    DEFAULT_OPTIONS = {
-      override_existing_file: false
-    }
+  class File::Destination < File::FileDestination
 
     def initialize(filename, options = {})
-      @filename = ::File.expand_path(filename)
-      @options = DEFAULT_OPTIONS.deep_merge(options)
-
-      if ::File.exists?(@filename) && @options[:override_existing_file] == false
-        raise "File `#{@filename}` exists but `override_existing_file` option was set to `false`"
-      end
-
-      @file = ::File.open(@filename, 'wb+')
-    end
-
-    def write(data)
-      return if data.blank?
-
-      if data.is_a?(Array)
-        data.each { |row| @file.write(row) }
-      else
-        @file.write(data)
-      end
-    end
-
-    def close
-      @file.close if @file
+      warn "[DEPRECATION] `Metacrunch::File::Destination` is deprecated.  Please use `Metacrunch::File::FileDestination` instead."
+      super
     end
 
   end

data/lib/metacrunch/file/file_destination.rb

@@ -0,0 +1,36 @@
+require "metacrunch/file"
+
+module Metacrunch
+  class File::FileDestination
+
+    DEFAULT_OPTIONS = {
+      override_existing_file: false
+    }
+
+    def initialize(filename, options = {})
+      @filename = ::File.expand_path(filename)
+      @options = DEFAULT_OPTIONS.deep_merge(options)
+
+      if ::File.exists?(@filename) && @options[:override_existing_file] == false
+        raise "File `#{@filename}` exists but `override_existing_file` option was set to `false`"
+      end
+
+      @file = ::File.open(@filename, 'wb+')
+    end
+
+    def write(data)
+      return if data.blank?
+
+      if data.is_a?(Array)
+        data.each { |row| @file.write(row) }
+      else
+        @file.write(data)
+      end
+    end
+
+    def close
+      @file.close if @file
+    end
+
+  end
+end

data/lib/metacrunch/file/file_source.rb

@@ -0,0 +1,56 @@
+require "metacrunch/file"
+require "rubygems/package"
+
+module Metacrunch
+  class File::FileSource
+
+    def initialize(filenames)
+      @filenames = [*filenames].map{|f| f.presence}.compact
+    end
+
+    def each(&block)
+      return enum_for(__method__) unless block_given?
+
+      @filenames.each do |filename|
+        if is_archive?(filename)
+         read_archive(filename, &block)
+        else
+         read_regular_file(filename, &block)
+        end
+      end
+    end
+
+  private
+
+    def is_archive?(filename)
+      filename.ends_with?(".tar") || filename.ends_with?(".tar.gz") || filename.ends_with?(".tgz")
+    end
+
+    def is_gzip_file?(filename)
+      filename.ends_with?(".gz") || filename.ends_with?(".tgz")
+    end
+
+    def read_regular_file(filename, &block)
+      if ::File.file?(filename)
+        io = is_gzip_file?(filename) ? Zlib::GzipReader.open(filename) : ::File.open(filename, "r")
+        yield File::Entry.new(filename: filename, archive_filename: nil, contents: io.read)
+      end
+    end
+
+    def read_archive(filename, &block)
+      io        = is_gzip_file?(filename) ? Zlib::GzipReader.open(filename) : ::File.open(filename, "r")
+      tarReader = Gem::Package::TarReader.new(io)
+
+      tarReader.each do |_tar_entry|
+        if _tar_entry.file?
+          yield File::Entry.new(
+            filename: filename,
+            archive_filename: _tar_entry.full_name,
+            contents: _tar_entry.read
+          )
+        end
+      end
+    end
+
+  end
+end

data/lib/metacrunch/file/source.rb

@@ -1,55 +1,11 @@
 require "metacrunch/file"
-require "rubygems/package"
 
 module Metacrunch
-  class File::Source
+  class File::Source < File::FileSource
 
     def initialize(filenames)
-      @filenames = [*filenames].map{|f| f.presence}.compact
-    end
-
-    def each(&block)
-      return enum_for(__method__) unless block_given?
-
-      @filenames.each do |filename|
-        if is_archive?(filename)
-         read_archive(filename, &block)
-        else
-         read_regular_file(filename, &block)
-        end
-      end
-    end
-
-  private
-
-    def is_archive?(filename)
-      filename.ends_with?(".tar") || filename.ends_with?(".tar.gz") || filename.ends_with?(".tgz")
-    end
-
-    def is_gzip_file?(filename)
-      filename.ends_with?(".gz") || filename.ends_with?(".tgz")
-    end
-
-    def read_regular_file(filename, &block)
-      if ::File.file?(filename)
-        io = is_gzip_file?(filename) ? Zlib::GzipReader.open(filename) : ::File.open(filename, "r")
-        yield File::Entry.new(filename: filename, archive_filename: nil, contents: io.read)
-      end
-    end
-
-    def read_archive(filename, &block)
-      io        = is_gzip_file?(filename) ? Zlib::GzipReader.open(filename) : ::File.open(filename, "r")
-      tarReader = Gem::Package::TarReader.new(io)
-
-      tarReader.each do |_tar_entry|
-        if _tar_entry.file?
-          yield File::Entry.new(
-            filename: filename,
-            archive_filename: _tar_entry.full_name,
-            contents: _tar_entry.read
-          )
-        end
-      end
+      warn "[DEPRECATION] `Metacrunch::File::Source` is deprecated.  Please use `Metacrunch::File::FileSource` instead."
+      super
     end
 
   end

data/lib/metacrunch/file/version.rb

@@ -1,5 +1,5 @@
 module Metacrunch
   module File
-    VERSION = "1.2.1"
+    VERSION = "1.3.0"
   end
 end

data/metacrunch-file.gemspec

@@ -16,5 +16,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "activesupport", ">= 5.1.0"
-  spec.add_dependency "axlsx", "~> 2.0.1"
+  spec.add_dependency "axlsx", ">= 3.0.0.pre"
+  spec.add_dependency "smarter_csv", "~> 1.2.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metacrunch-file
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.3.0
 platform: ruby
 authors:
 - René Sprotte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-18 00:00:00.000000000 Z
+date: 2019-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -26,18 +26,32 @@ dependencies:
         version: 5.1.0
 - !ruby/object:Gem::Dependency
   name: axlsx
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0.pre
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0.pre
+- !ruby/object:Gem::Dependency
+  name: smarter_csv
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.1
+        version: 1.2.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.1
+        version: 1.2.6
 description: 
 email: 
 executables: []
@@ -53,8 +67,11 @@ files:
 - Readme.md
 - bin/console
 - lib/metacrunch/file.rb
+- lib/metacrunch/file/csv_source.rb
 - lib/metacrunch/file/destination.rb
 - lib/metacrunch/file/entry.rb
+- lib/metacrunch/file/file_destination.rb
+- lib/metacrunch/file/file_source.rb
 - lib/metacrunch/file/source.rb
 - lib/metacrunch/file/version.rb
 - lib/metacrunch/file/xlsx_destination.rb
@@ -78,8 +95,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: File package for the metacrunch ETL toolkit.