metacrunch-file 1.1.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: f457f5fcca1eb193efe81c0e7c8f02d73b42bc7e
-  data.tar.gz: 64f70e6bf5a0d45014853bbae5d09e75456a67c1
+SHA256:
+  metadata.gz: 1a64cc33b8badb7d4e6b7c29267501db505edbb081ed55be3c4bdf269109eac8
+  data.tar.gz: 0bb18d835fb8e812ff04aded3b8bfc3d90f8daae606704c49d2dbfe502cdaafd
 SHA512:
-  metadata.gz: 80edcd6aaa76b570371fd14b6bbe59a50df168e2471e265075b5607504ae8270d33a2787dfab48bf038d30d8d8261b1c178d7ec306f127873b2bac1a654f64ac
-  data.tar.gz: 3ab68b17ab6f7971e286d6177db0c80a4d1b8a83d1d97b707e1ee188e1af55eb2ba1239bc2d47262abc190f7f52503418577e62e5ec6707f4e5d188b13e77197
+  metadata.gz: a28ff6fcb40f286673011ee27de86a15d579bdc213b379eb77781eddbd6efa5613e3d68bd16e1d016d61f9d786b143c35e2b276961affbad21d71707060c65a4
+  data.tar.gz: e58ec6254b11f956c4a45a0ef68477cee7b9bb2e79ce5fd283f5a3e0450e0e4b49c5c53796f7e6234ad9742d46fd42e19fed6bd2be231847b8de6a82cb188d57

data/.circleci/config.yml

@@ -1,12 +1,11 @@
-# Ruby CircleCI 2.0 configuration file
-#
-# Check https://circleci.com/docs/2.0/language-ruby/ for more details
-#
-version: 2
+version: 2.1
+orbs:
+  ruby: circleci/ruby@1.1.1
+
 jobs:
   build:
     docker:
-      - image: circleci/ruby:2.4.1-node-browsers
+      - image: circleci/ruby:2.6-node-browsers
 
     working_directory: ~/repo
 

data/Gemfile

@@ -14,6 +14,6 @@ end
 group :test do
   gem "rspec",                 ">= 3.5.0", "< 4.0.0"
   gem "rspec_junit_formatter", ">= 0.3.0"
-  gem "simplecov",             ">= 0.15.0"
+  gem "simplecov",             "= 0.17.1"
 end
 

data/Readme.md

@@ -6,7 +6,10 @@ metacrunch-file
 [![Test Coverage](https://codeclimate.com/github/ubpb/metacrunch-file/badges/coverage.svg)](https://codeclimate.com/github/ubpb/metacrunch-file/coverage)
 [![CircleCI](https://circleci.com/gh/ubpb/metacrunch-file.svg?style=svg)](https://circleci.com/gh/ubpb/metacrunch-file)
 
-This is the official file package for the [metacrunch ETL toolkit](https://github.com/ubpb/metacrunch).
+This is the official file package for the [metacrunch ETL toolkit](https://github.com/ubpb/metacrunch). 
+
+*Note: For working examples on how to use this package check out our [demo repository](https://github.com/ubpb/metacrunch-demo).*
+
 
 Installation
 ------------
@@ -14,7 +17,7 @@ Installation
 Include the gem in your `Gemfile`
 
 ```ruby
-gem "metacrunch-file", "~> 1.0.0"
+gem "metacrunch-file", "~> 1.4.0"
 ```
 
 and run `$ bundle install` to install it.
@@ -29,24 +32,20 @@ $ gem install metacrunch-file
 Usage
 -----
 
-*Note: For working examples on how to use this package check out our [demo repository](https://github.com/ubpb/metacrunch-demo).*
-
-### `Metacrunch::File::Source`
+## `Metacrunch::File::FileSource`
 
 This class provides a metacrunch `source` implementation that can be used to read data from files in the file system into a metacrunch job. The class can be used to read regular files, compressed files (gzip), tar archives and compressed tar archives (gzip).
 
-You can access non-option arguments from the command line using the `ARGV` constant.
-
 ```ruby
 # my_job.metacrunch
 
 # If you call this example like so
 #   $ metacrunch my_job.metacrunch *.xml
 # ARGV will contain all the XML files in the current directory.
-source Metacrunch::File::Source.new(ARGV)
+source Metacrunch::File::FileSource.new(ARGV)
 
 # ... or you can set the filenames directly
-source Metacrunch::File::Source.new(["my-data.xml", "my-other-data.xml", "..."])
+source Metacrunch::File::FileSource.new(["my-data.xml", "my-other-data.xml", "..."])
 ```
 
 **Options**
@@ -67,12 +66,43 @@ transformation ->(file_entry) do
 end
 ```
 
+## `Metacrunch::File::FileDestination`
+
+This class provides a metacrunch `destination` to write data to a file. Every data that gets passed to the destination is appended to the given file. If the data is an `Array` every element of that array is appended to the file. Non existing files will be created automatically.
+
+```ruby
+# my_job.metacrunch
+
+destination Metacrunch::File::FileDestination.new("/tmp/my-data.txt" [, OPTIONS])
+```
+
+**Options**
+
+* `override_existing_file`: Overrides an existing file if set to `true`. If set to `false` an error is raised if the file already exists. Defaults to `false`.
+
+## `Metacrunch::File::CSVSource`
+
+This class provides a metacrunch `source` for reading CSV files. It is a simple wrapper around [smarter_csv](https://github.com/tilo/smarter_csv) gem. 
+
+```ruby
+# my_job.metacrunch
+
+source Metacrunch::File::CSVSource.new("my.csv" [, OPTIONS])
+```
+
+**Options**
+
+* `headers`: Whether or not the file contains headers as the first line. Important if the file does not contain headers, otherwise you would lose the first line of data. Defaults to `true`.
+* `col_sep`: Column separator. Defaults to `,`.
+* `row_sep`: Row separator or record separator. Defaults to `\n`.
+* `quote_char`: Quotation character. Defaults to `"`.
+* `file_encoding`: Set the file encoding. Defaults to `utf-8`.
 
-### `Metacrunch::File::XLSXDestination`
+## `Metacrunch::File::XLSXDestination`
 
 This class provides a metacrunch `destination` implementation to create simple Excel (xlsx) files.
 
-To use this destination a transformation is required to format the data in a proper array that can be feet into the destination. When defining the destination you must provide an array of column names. Each data row passed to the destination must be an array of the same size as the column array.
+To use this destination a transformation is required to format the data in a proper array that can be passed to the destination. When defining the destination you must provide an array of column names. Each data row passed to the destination must be an array of the same size as the column array.
 
 ```ruby
 # my_job.metacrunch
@@ -81,7 +111,7 @@ transformation ->(data) do
   [data["foo"], data["bar"], ...]
 end
 
-source Metacrunch::File::XLSXDestination.new(
+destination Metacrunch::File::XLSXDestination.new(
     "/tmp/my-data.xlsx",           # filename
     ["Column 1", "Column 2", ...], # header columns
     OPTIONS

data/lib/metacrunch/file.rb

@@ -4,7 +4,11 @@ require "active_support/core_ext"
 module Metacrunch
   module File
     require_relative "file/entry"
+    require_relative "file/csv_source"
+    require_relative "file/file_source"
     require_relative "file/source"
+    require_relative "file/file_destination"
+    require_relative "file/destination"
     require_relative "file/xlsx_destination"
   end
 end

data/lib/metacrunch/file/csv_source.rb

@@ -0,0 +1,35 @@
+require "metacrunch/file"
+require "smarter_csv"
+
+module Metacrunch
+  class File::CSVSource
+
+    DEFAULT_OPTIONS = {
+      headers: true,
+      col_sep: ",",
+      row_sep: "\n",
+      quote_char: '"',
+      file_encoding: "utf-8"
+    }
+
+    def initialize(csv_filename, options = {})
+      @filename = csv_filename
+      @options = DEFAULT_OPTIONS.merge(options)
+    end
+
+    def each(&block)
+      return enum_for(__method__) unless block_given?
+
+      SmarterCSV.process(@filename, {
+        headers_in_file: @options[:headers],
+        col_sep: @options[:col_sep],
+        row_sep: @options[:row_sep],
+        quote_char: @options[:quote_char],
+        file_encoding: @options[:file_encoding]
+      }) do |line|
+        yield line
+      end
+    end
+
+  end
+end

data/lib/metacrunch/file/destination.rb

@@ -0,0 +1,12 @@
+require "metacrunch/file"
+
+module Metacrunch
+  class File::Destination < File::FileDestination
+
+    def initialize(filename, options = {})
+      warn "[DEPRECATION] `Metacrunch::File::Destination` is deprecated.  Please use `Metacrunch::File::FileDestination` instead."
+      super
+    end
+
+  end
+end

data/lib/metacrunch/file/file_destination.rb

@@ -0,0 +1,36 @@
+require "metacrunch/file"
+
+module Metacrunch
+  class File::FileDestination
+
+    DEFAULT_OPTIONS = {
+      override_existing_file: false
+    }
+
+    def initialize(filename, options = {})
+      @filename = ::File.expand_path(filename)
+      @options = DEFAULT_OPTIONS.deep_merge(options)
+
+      if ::File.exists?(@filename) && @options[:override_existing_file] == false
+        raise "File `#{@filename}` exists but `override_existing_file` option was set to `false`"
+      end
+
+      @file = ::File.open(@filename, 'wb+')
+    end
+
+    def write(data)
+      return if data.blank?
+
+      if data.is_a?(Array)
+        data.each { |row| @file.write(row) }
+      else
+        @file.write(data)
+      end
+    end
+
+    def close
+      @file.close if @file
+    end
+
+  end
+end

data/lib/metacrunch/file/file_source.rb

@@ -0,0 +1,56 @@
+require "metacrunch/file"
+require "rubygems/package"
+
+module Metacrunch
+  class File::FileSource
+
+    def initialize(filenames)
+      @filenames = [*filenames].map{|f| f.presence}.compact
+    end
+
+    def each(&block)
+      return enum_for(__method__) unless block_given?
+
+      @filenames.each do |filename|
+        if is_archive?(filename)
+         read_archive(filename, &block)
+        else
+         read_regular_file(filename, &block)
+        end
+      end
+    end
+
+  private
+
+    def is_archive?(filename)
+      filename.ends_with?(".tar") || filename.ends_with?(".tar.gz") || filename.ends_with?(".tgz")
+    end
+
+    def is_gzip_file?(filename)
+      filename.ends_with?(".gz") || filename.ends_with?(".tgz")
+    end
+
+    def read_regular_file(filename, &block)
+      if ::File.file?(filename)
+        io = is_gzip_file?(filename) ? Zlib::GzipReader.open(filename) : ::File.open(filename, "r")
+        yield File::Entry.new(filename: filename, archive_filename: nil, contents: io.read)
+      end
+    end
+
+    def read_archive(filename, &block)
+      io        = is_gzip_file?(filename) ? Zlib::GzipReader.open(filename) : ::File.open(filename, "r")
+      tarReader = Gem::Package::TarReader.new(io)
+
+      tarReader.each do |_tar_entry|
+        if _tar_entry.file?
+          yield File::Entry.new(
+            filename: filename,
+            archive_filename: _tar_entry.full_name,
+            contents: _tar_entry.read
+          )
+        end
+      end
+    end
+
+  end
+end

data/lib/metacrunch/file/source.rb

@@ -1,55 +1,11 @@
 require "metacrunch/file"
-require "rubygems/package"
 
 module Metacrunch
-  class File::Source
+  class File::Source < File::FileSource
 
     def initialize(filenames)
-      @filenames = [*filenames].map{|f| f.presence}.compact
-    end
-
-    def each(&block)
-      return enum_for(__method__) unless block_given?
-
-      @filenames.each do |filename|
-        if is_archive?(filename)
-         read_archive(filename, &block)
-        else
-         read_regular_file(filename, &block)
-        end
-      end
-    end
-
-  private
-
-    def is_archive?(filename)
-      filename.ends_with?(".tar") || filename.ends_with?(".tar.gz") || filename.ends_with?(".tgz")
-    end
-
-    def is_gzip_file?(filename)
-      filename.ends_with?(".gz") || filename.ends_with?(".tgz")
-    end
-
-    def read_regular_file(filename, &block)
-      if ::File.file?(filename)
-        io = is_gzip_file?(filename) ? Zlib::GzipReader.open(filename) : ::File.open(filename, "r")
-        yield File::Entry.new(filename: filename, archive_filename: nil, contents: io.read)
-      end
-    end
-
-    def read_archive(filename, &block)
-      io        = is_gzip_file?(filename) ? Zlib::GzipReader.open(filename) : ::File.open(filename, "r")
-      tarReader = Gem::Package::TarReader.new(io)
-
-      tarReader.each do |_tar_entry|
-        if _tar_entry.file?
-          yield File::Entry.new(
-            filename: filename,
-            archive_filename: _tar_entry.full_name,
-            contents: _tar_entry.read
-          )
-        end
-      end
+      warn "[DEPRECATION] `Metacrunch::File::Source` is deprecated.  Please use `Metacrunch::File::FileSource` instead."
+      super
     end
 
   end

data/lib/metacrunch/file/version.rb

@@ -1,6 +1,5 @@
-
 module Metacrunch
   module File
-    VERSION = "1.1.0"
+    VERSION = "1.4.0"
   end
 end

data/lib/metacrunch/file/xlsx_destination.rb

@@ -20,9 +20,17 @@ module Metacrunch
       @sheet.add_row(columns, types: :string)
     end
 
-    def write(row)
-      return if row.blank?
-      @sheet.add_row(row, types: :string)
+    def write(data)
+      return if data.blank?
+      raise ArgumentError, "Data must be an Array" unless data.is_a?(Array)
+
+      if data.first.is_a?(Array)
+        data.each do |d|
+          @sheet.add_row(d, types: :string)
+        end
+      else
+        @sheet.add_row(data, types: :string)
+      end
     end
 
     def close

data/metacrunch-file.gemspec

@@ -15,6 +15,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "activesupport", ">= 5.1.0"
-  spec.add_dependency "axlsx", "~> 2.0.1"
+  spec.add_dependency "activesupport", ">= 0"
+  spec.add_dependency "caxlsx", "~> 3.0"
+  spec.add_dependency "smarter_csv", "~> 1.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metacrunch-file
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.4.0
 platform: ruby
 authors:
 - René Sprotte
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-08 00:00:00.000000000 Z
+date: 2020-10-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -16,30 +16,44 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.1.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.1.0
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: axlsx
+  name: caxlsx
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.1
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.1
-description: 
-email: 
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: smarter_csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+description:
+email:
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -53,7 +67,11 @@ files:
 - Readme.md
 - bin/console
 - lib/metacrunch/file.rb
+- lib/metacrunch/file/csv_source.rb
+- lib/metacrunch/file/destination.rb
 - lib/metacrunch/file/entry.rb
+- lib/metacrunch/file/file_destination.rb
+- lib/metacrunch/file/file_source.rb
 - lib/metacrunch/file/source.rb
 - lib/metacrunch/file/version.rb
 - lib/metacrunch/file/xlsx_destination.rb
@@ -62,7 +80,7 @@ homepage: http://github.com/ubpb/metacrunch-file
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -77,9 +95,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: File package for the metacrunch ETL toolkit.
 test_files: []