csvpp 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 96198044193b9b7a62d2e35810217bfc5b603c7b
+  data.tar.gz: c5f025bd90f45294986eb3892c5db6d81f2b5498
+SHA512:
+  metadata.gz: 3b35fd7ed568851367faf5b16c329271fa4d7d5935161037ceb1e8fba739d84515c8fec63c0382394156914e3d6eb2d4116b3c75ed0a88986e6554487b3b16db
+  data.tar.gz: 211dc3241614ba542156221574054264528a2baed9f99c8875c3e01d041d0a04065723cb5bc5e4cf04f539822131c96ed92789fdf3d4c72c152cbab55d7a0830

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+.idea

data/.gitlab-ci.yml

@@ -0,0 +1,13 @@
+image: "ruby:2.4"
+
+cache:
+  paths:
+    - vendor/
+
+before_script:
+  - gem install bundler --no-ri --no-rdoc
+  - bundle install -j $(nproc) --path vendor
+
+tests:
+  script:
+    - bundle exec rake

data/.ruby-version

@@ -0,0 +1 @@
+2.4.0

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.2
+before_install: gem install bundler -v 1.16.0.pre.3

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in csvpp.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,51 @@
+PATH
+  remote: .
+  specs:
+    csvpp (0.3.0)
+      oj (~> 3.3.9)
+      sinatra (~> 2.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    byebug (9.1.0)
+    coderay (1.1.2)
+    method_source (0.9.0)
+    minitest (5.10.3)
+    mustermann (1.0.1)
+    oj (3.3.10)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    pry-byebug (3.5.0)
+      byebug (~> 9.1)
+      pry (~> 0.10)
+    pry-doc (0.11.1)
+      pry (~> 0.9)
+      yard (~> 0.9)
+    rack (2.0.3)
+    rack-protection (2.0.0)
+      rack
+    rake (12.3.0)
+    sinatra (2.0.0)
+      mustermann (~> 1.0)
+      rack (~> 2.0)
+      rack-protection (= 2.0.0)
+      tilt (~> 2.0)
+    tilt (2.0.8)
+    yard (0.9.11)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler
+  csvpp!
+  minitest
+  pry
+  pry-byebug
+  pry-doc
+  rake
+
+BUNDLED WITH
+   1.14.6

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Rathesan Iyadurai
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,47 @@
+# CSV++
+
+CSV++ takes a `<DELIMITER>` separated input file and a JSON format specification
+and turns it into Ruby Objects. See `test/sample_inputs/simple.txt` and
+`test/sample_formats/simple.json` for example.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'csvpp'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install csvpp
+
+## Usage
+
+```ruby
+CSVPP.parse(
+  input: 'test/sample_inputs/simple.txt',
+  format: 'test/sample_formats/simple.json'
+) # => [{"v1"=>34, "line_number"=>1, "v2"=>"foobar"}, {"v1"=>99, "line_number"=>2, "v2"=>"hi  there"}]
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run
+`rake test` to run the tests. You can also run `bin/console` for an interactive
+prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To
+release a new version, update the version number in `version.rb`, and then run
+`bundle exec rake release`, which will create a git tag for the version, push
+git commits and tags, and push the `.gem` file to
+[rubygems.org](https://rubygems.org).
+
+## License
+
+The gem is available as open source under the terms of the [MIT
+License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "csvpp"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "pry"
+Pry.start(CSVPP)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/config.ru

@@ -0,0 +1,4 @@
+require 'csvpp'
+require 'csvpp/api'
+
+run CSVPP::API

data/csvpp.gemspec

@@ -0,0 +1,42 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "csvpp/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "csvpp"
+  spec.version       = CSVPP::VERSION
+  spec.authors       = ["SwissDRG AG"]
+  spec.email         = ["rathesan.iyadurai@swissdrg.org"]
+
+  spec.summary       = %q{CSV++}
+  spec.description   = %q{CSV++}
+  spec.homepage      = "https://www.swissdrg.org"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  # if spec.respond_to?(:metadata)
+  #   spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
+  # else
+  #   raise "RubyGems 2.0 or newer is required to protect against " \
+  #     "public gem pushes."
+  # end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'sinatra', '~> 2.0'
+  spec.add_dependency 'oj', '~> 3.3.9'
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "minitest"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "pry-byebug"
+  spec.add_development_dependency "pry-doc"
+end

data/exe/csvpp

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require_relative '../lib/csvpp'
+
+$options = {}
+option_parser = OptionParser.new do |opts|
+  opts.on('-f', '--format FORMAT') do |format|
+    $options[:format] = format.strip
+  end
+
+  opts.on_tail('-h', '--help', 'Show this message') do
+    puts opts
+    exit
+  end
+end
+
+HELP = option_parser.help
+option_parser.parse!
+
+format = $options[:format]
+
+json = CSVPP.json(
+  input: ARGF.read,
+  format: File.read(format)
+)
+
+# Forgive me Matz for I have sinned
+system %{echo '#{json}' | python -m json.tool}

data/lib/csvpp/api.rb

@@ -0,0 +1,25 @@
+require 'sinatra/base'
+
+module CSVPP
+  class API < Sinatra::Base
+    before do
+      headers 'Access-Control-Allow-Origin' => '*'
+
+      content_type :json
+    end
+
+    post '/parse' do
+      body = Oj.load(request.body.read)
+      input = body.fetch('input')
+      format = body.fetch('format')
+
+      CSVPP.json(input: input, format: format)
+    end
+
+    options "*" do
+      response.headers["Allow"] = "HEAD,GET,PUT,POST,DELETE,OPTIONS"
+      response.headers["Access-Control-Allow-Headers"] = "X-Requested-With, X-HTTP-Method-Override, Content-Type, Cache-Control, Accept"
+      200
+    end
+  end
+end

data/lib/csvpp/conversions.rb

@@ -0,0 +1,102 @@
+require 'bigdecimal'
+
+module CSVPP
+  module Conversions
+    module_function
+
+    # @param obj [Object] object to parse
+    # @param to [String] a type, e.g. "int"
+    # @missings [Array] list of values that are treated as missings, e.g. ['NA', '-', -999]
+    # @params options [Hash] options passed on to parsing methods for specific types
+    # @return parsed value, read from `obj`, interpreted as type given by `to`
+    def convert(obj, to:, missings: [], **options)
+      return nil if missing?(obj, missings)
+
+      send("parse_#{to}", obj, **options)
+    end
+
+    def parse_string(str, **options)
+      str.to_s
+    end
+
+    def parse_int(str, **options)
+      return nil if str.to_s.empty?
+
+      cleaned = if str.is_a?(String)
+                  val = str.strip
+                          .gsub(/['`\s]?/, '')      # remove thousand separators
+                          .sub(/\.\d*/, '')         # remove decimal point and everything thereafter
+                          .sub(/[\sa-zA-Z]*$/, '')  # remove trailing words like "mg"
+                          .sub(/^-0*(.+)$/, '-\1')  # remove 0 after negative sign: -003 => -3
+                  val =~ /^0+$/ ? '0' : val.gsub( /^0*/, '')      # remove leading zeros
+                else
+                  str
+                end
+      Integer(cleaned) rescue nil
+    end
+
+    def parse_float(str, **options)
+      return nil if str.to_s.empty?
+      Float(clean_decimal(str)) rescue nil
+    end
+
+    def parse_decimal(str)
+      return nil if str.to_s.empty?
+
+      cleaned = clean_decimal(str).to_s
+
+      if cleaned.empty?
+        nil
+      else
+        BigDecimal(cleaned)
+      end
+
+    end
+
+    def parse_date(str, **options)
+      Date.parse(str.to_s)
+    end
+
+    # @param true_values [Array]: list of values that are interpreted as `true`
+    # @param false_values [Array]: list of values that are interpreted as `false`
+    # @return true or false, or
+    #      nil if `str` doesn't match any value interpreted as `true` or `false`
+    def parse_boolean(str, true_values: [], false_values: [])
+      cleaned = str.to_s.strip.downcase
+
+      trues = if true_values.empty?
+                ['1', 't', 'true']
+              else
+                true_values.map(&:to_s).map(&:downcase)
+              end
+      return true if trues.include? cleaned
+
+      falses = if false_values.empty?
+                 ['0', 'f', 'false']
+               else
+                 false_values.map(&:to_s).map(&:downcase)
+               end
+      return false if falses.include? cleaned
+
+      nil
+    end
+
+    def missing?(obj, missings)
+      missings.map(&:to_s).include?(obj.to_s)
+    end
+
+    def clean_decimal(str)
+      return str unless str.is_a?(String)
+
+      val = str.strip
+               .gsub(/['`\s]?/, '')               # remove thousand separators
+               .sub(/[\sa-zA-Z]*$/, '')           # remove trailing words like "mg"
+               .sub(/^-0*(.+)$/, '-\1')           # remove 0 after negative sign: -003 => -3
+      if val =~ /^0+$/                            # remove leading zeros
+        '0'
+      else
+        val.gsub( /^0*/, '')
+      end
+    end
+  end
+end

data/lib/csvpp/format.rb

@@ -0,0 +1,126 @@
+module CSVPP
+  class Format
+    attr_reader :name, :skip
+
+    class << self
+      # @param name [String] unique name of the format
+      # @param format [Format]
+      def add(name, format)
+        store[name] = format
+      end
+
+      # @param name [String] unique name of the format
+      def find(name)
+        store.fetch(name)
+      end
+
+      # @param path [String] path to format file
+      # @return [Format]
+      def load(path)
+        load_from_str File.read(path)
+      end
+
+      # @param json [String]
+      # @return [Format]
+      def load_from_str(json)
+        new Oj.load(json)
+      end
+
+      def all
+        store.values
+      end
+
+      def store
+        @store ||= {}
+      end
+    end
+
+    # @param format [Hash]
+    def initialize(format)
+      @name = format['name']
+      @multiline = format['multiline'].to_s.strip.downcase == 'true'
+      @skip = format['skip'].to_i
+      @vars = format.fetch('vars')
+
+      if multiline?
+        @vars_grouped_by_line = Hash[
+          vars.group_by { |var, meta| meta['line'] }.map do |line_id, vars|
+            [line_id, vars.map { |var, *| var }]
+          end
+        ]
+
+        @multiline_start = format.fetch('start')
+      end
+    end
+
+    def var_names
+      vars.keys
+    end
+
+    def length
+      var_names.count
+    end
+
+    def index(var)
+      position(var) - 1
+    end
+
+    def position(var)
+      vars.fetch(var)['position']
+    end
+
+    def type(var)
+      vars.fetch(var)['type']
+    end
+
+    # @param var [String]: name of the variable for which the missings are required
+    # @return [Array] an array of missing values (can be empty if no missings were defined)
+    def missings(var)
+      array_from(var, 'missings')
+    end
+
+    # Returns the values that are defined as `true` in the the format's json
+    # definition for the given variable.
+    # @return [Array] all values that should be interpreted as true for this variable
+    def true_values(var)
+      return [] unless type(var) == "boolean"
+      array_from(var, 'true_values')
+    end
+
+
+    # Returns the values that are defined as `false` in the the format's json
+    # definition for the given variable.
+    # @return [Array] all values that should be interpreted as `false` for this variable
+    def false_values(var)
+      return [] unless type(var) == "boolean"
+      array_from(var, 'false_values')
+    end
+
+    def vars_for_line(line_id)
+      vars_grouped_by_line.fetch(line_id)
+    end
+
+    def multiline_start?(line_id)
+      multiline_start == line_id
+    end
+
+    def multiline?
+      @multiline
+    end
+
+    private
+
+    # Returns the value or values specified for the given attribute of the given
+    # variable in the formats json. An empty array if no such attribute was
+    # defined for the given variable.
+    # @return [Array] value(s) defined for given attribute for given variable
+    def array_from(var, attribute)
+      value = vars.fetch(var)[attribute]
+      return [] if value.nil?
+      return value if value.is_a?(Array)
+      [value]
+    end
+
+    attr_reader :vars, :vars_grouped_by_line, :multiline_start
+  end
+end