csvpp 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 880efa52b42f34d49e6aeeb59cfb05211f274ff6
-  data.tar.gz: 226688ad42678fdca218b2a2fcd047d08846aa62
+SHA256:
+  metadata.gz: f70243c72416b66f569849a07a20845019889f990cfb3b6ad3859886b0c5a387
+  data.tar.gz: da8124bf9974a3ebc0855c9a799e8b17c55c139879c46600c9597712af7c4a61
 SHA512:
-  metadata.gz: 6b030f48cec237f995f2b1fec2912a0229cd6e0b224bbdac5cc007ff7db2121f0d7f8e0c79fae96c4eeabbb54d9f4c8b541bb410a60dbe632e4f1ef049bae795
-  data.tar.gz: 72c4d7c77acb0d0bffb903f1236e031434d5b8116ce73167f0be94f6b58d5033944427fca2c5fc39ec1e45f07ecead446f4375833e29b6e2e87336303a323415
+  metadata.gz: f33f39b8ef836fd0fa54af8f5a185095bb0daabd060476c1f71c790b28606e3009e2a1aa69ba2ffdd0652ce9a84ce007e5d5c13757afd7d209868bd52b932402
+  data.tar.gz: 907a4e1d8fc6a1933629632a4fc5a2ac724f6d3f1498e769639e598ca6b749e9c03a4ab1d0099c56d5e89491a788e0d5c91ac27dd2716d7c9a6449e49f7bda41

data/.gitignore

@@ -8,3 +8,4 @@
 /tmp/
 
 .idea
+*.db

data/.gitlab-ci.yml

@@ -1,4 +1,4 @@
-image: "ruby:2.4"
+image: "ruby:2.5"
 
 cache:
   paths:

data/.rubocop.yml

@@ -0,0 +1,78 @@
+AllCops:
+  DisplayCopNames: true
+  DisplayStyleGuide: true
+  TargetRubyVersion: 2.5
+  Include:
+    - Rakefile
+    - config.ru
+  Exclude:
+    - Gemfile
+    - csvpp.gemspec
+
+Metrics/LineLength:
+  Max: 80
+  Exclude:
+    - test/**/**
+
+Style/ClassAndModuleChildren:
+  Enabled: false
+
+Style/PercentLiteralDelimiters:
+  Enabled: false
+
+Style/NumericLiteralPrefix:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/EmptyMethod:
+  Enabled: false
+
+Style/DoubleNegation:
+  Enabled: false
+
+Style/WordArray:
+  Exclude:
+    - test/**/*
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/ModuleLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/CyclomaticComplexity:
+  Enabled: false
+
+Metrics/PerceivedComplexity:
+  Enabled: false
+
+Metrics/AbcSize:
+  Enabled: false
+
+Naming/HeredocDelimiterNaming:
+  Enabled: false
+
+Layout/MultilineMethodCallIndentation:
+  EnforcedStyle: indented_relative_to_receiver
+
+Layout/AlignHash:
+  Exclude:
+    - lib/csvpp/cli.rb
+
+Lint/UnusedBlockArgument:
+  Exclude:
+    - lib/csvpp/cli.rb
+    - lib/csvpp/conversions.rb
+
+Lint/UnusedMethodArgument:
+  Exclude:
+    - lib/csvpp/cli.rb
+    - lib/csvpp/conversions.rb

data/.ruby-version

@@ -1 +1 @@
-2.4.0
+2.5.1

data/Gemfile.lock

@@ -1,17 +1,27 @@
 PATH
   remote: .
   specs:
-    csvpp (0.3.1)
-      oj (~> 3.3.9)
+    csvpp (0.4.0)
+      gli (~> 2.17.1)
+      httparty (~> 0.16.2)
+      oj (~> 3.6.2)
+      sequel (~> 5.9)
+      sqlite3 (~> 1.3, >= 1.3.13)
 
 GEM
   remote: https://rubygems.org/
   specs:
     byebug (9.1.0)
     coderay (1.1.2)
+    docile (1.3.1)
+    gli (2.17.1)
+    httparty (0.16.2)
+      multi_xml (>= 0.5.2)
+    json (2.1.0)
     method_source (0.9.0)
     minitest (5.10.3)
-    oj (3.3.10)
+    multi_xml (0.6.0)
+    oj (3.6.3)
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
@@ -22,6 +32,14 @@ GEM
       pry (~> 0.9)
       yard (~> 0.9)
     rake (12.3.0)
+    ruby-prof (0.16.2)
+    sequel (5.9.0)
+    simplecov (0.16.1)
+      docile (~> 1.1)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.2)
+    sqlite3 (1.3.13)
     yard (0.9.11)
 
 PLATFORMS
@@ -35,6 +53,8 @@ DEPENDENCIES
   pry-byebug
   pry-doc
   rake
+  ruby-prof
+  simplecov
 
 BUNDLED WITH
-   1.14.6
+   1.16.2

data/README.md

@@ -1,5 +1,8 @@
 # CSV++
 
+![build](http://gitlab/gems/csvpp/badges/master/build.svg)
+![coverage](http://gitlab/gems/csvpp/badges/master/coverage.svg)
+
 CSV++ takes a `<DELIMITER>` separated input file and a JSON format specification
 and turns it into Ruby Objects. See `test/sample_inputs/simple.txt` and
 `test/sample_formats/simple.json` for example.
@@ -24,6 +27,15 @@ Or install it yourself as:
 $ gem install csvpp
 ```
 
+### For development of CSV++
+
+First make sure that you have bundler and the Ruby version installed that is
+specified in `.ruby-version`. Then run:
+
+```
+$ bundle
+```
+
 ## Usage
 
 ```ruby
@@ -35,39 +47,26 @@ CSVPP.parse(
 
 ### CLI
 
-CSV++ comes with a CLI. To print a JSON representation of an input file, pass a
-file argument and provide a format specification file with `-f`, e.g.:
+CSV++ comes with a CLI that speaks with the [Formats API](http://formats.iapps.swissdrg.local/api/formats).
+
+To print a list of known formats run:
 
 ```
-$ csvpp test/sample_inputs/simple.txt -f test/sample_formats/simple.json
-
-{
-    "vars": [
-        {
-            "line_number": 1,
-            "v1": 34,
-            "v2": "foobar",
-            "v3": 1.1,
-            "v4": false
-        },
-        {
-            "line_number": 2,
-            "v1": 99,
-            "v2": "hi  there",
-            "v3": 2.2,
-            "v4": true
-        },
-        {
-            "line_number": 3,
-            "v1": null,
-            "v2": "Missing",
-            "v3": null,
-            "v4": true
-        }
-    ]
-}
+$ csvpp formats
 ```
 
+The `parse` command can be used to parse an input file with a given format,
+either fetched from the API or from a local JSON specification file.
+
+For example, to import some REKOLE cost data to a local sqlite DB for querying,
+run:
+
+```
+$ csvpp parse --format fk_2017 --output costs.db test/sample_inputs/fk_2017.txt
+```
+
+Run `csvpp help parse` for more details.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run

data/bin/profile

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'csvpp'
+
+require 'ruby-prof'
+
+f = CSVPP::Format.load(ARGV[0])
+result = RubyProf.profile do
+  CSVPP::Parser.parse(
+    input: ARGV[1],
+    format: f,
+    convert_type: false
+  )
+end
+
+printer = RubyProf::FlatPrinter.new(result)
+printer.print(STDOUT)

data/csvpp.gemspec

@@ -30,7 +30,11 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency 'oj', '~> 3.3.9'
+  spec.add_dependency 'oj', '~> 3.6.2'
+  spec.add_dependency 'gli', '~> 2.17.1'
+  spec.add_dependency 'httparty', '~> 0.16.2'
+  spec.add_dependency 'sqlite3', '~> 1.3', '>= 1.3.13'
+  spec.add_dependency 'sequel', '~> 5.9'
 
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
@@ -38,4 +42,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "pry"
   spec.add_development_dependency "pry-byebug"
   spec.add_development_dependency "pry-doc"
+  spec.add_development_dependency "ruby-prof"
+  spec.add_development_dependency "simplecov"
 end

data/exe/csvpp

@@ -1,29 +1,4 @@
 #!/usr/bin/env ruby
 
-require 'optparse'
-require_relative '../lib/csvpp'
-
-$options = {}
-option_parser = OptionParser.new do |opts|
-  opts.on('-f', '--format FORMAT') do |format|
-    $options[:format] = format.strip
-  end
-
-  opts.on_tail('-h', '--help', 'Show this message') do
-    puts opts
-    exit
-  end
-end
-
-HELP = option_parser.help
-option_parser.parse!
-
-format = $options[:format]
-
-json = CSVPP.json(
-  input: ARGF.read,
-  format: File.read(format)
-)
-
-# Forgive me Matz for I have sinned
-system %{echo '#{json}' | python -m json.tool}
+require_relative '../lib/csvpp/cli'
+exit CSVPP::CLI.run(ARGV)

data/lib/csvpp.rb

@@ -1,49 +1,77 @@
+# frozen_string_literal: true
+
 require 'oj'
+Oj.default_options = { mode: :compat }
 
 require_relative './csvpp/version'
+require_relative './csvpp/os'
 require_relative './csvpp/conversions'
 require_relative './csvpp/format'
 require_relative './csvpp/parser'
+require_relative './csvpp/sqlite_importer'
+require_relative './csvpp/formats_client'
 
 module CSVPP
-
   DEFAULT_COL_SEP = '|'
 
   # @param input [String] path to input file
-  # @param format [String] path to format file
+  # @param format [String, Format] path to format file
   # @param col_sep [String]
   #
   # @return [Array<Object>]
-  def self.parse(input:, format:, col_sep: DEFAULT_COL_SEP, &block)
+  def self.parse(input:,
+                 format:,
+                 col_sep: DEFAULT_COL_SEP,
+                 convert_type: true,
+                 &block)
+
     Parser.parse(
       input: input,
       format: Format.load(format),
       col_sep: col_sep,
+      convert_type: convert_type,
       &block
     )
   end
 
   # @param input [String] input string
-  # @param format [String] format string
+  # @param format [String, Format] format string
   # @param col_sep [String]
   #
   # @return [Array<Object>]
-  def self.parse_str(input:, format:, col_sep: DEFAULT_COL_SEP, &block)
+  def self.parse_str(input:,
+                     format:,
+                     col_sep: DEFAULT_COL_SEP,
+                     convert_type: true,
+                     &block)
+
     Parser.parse_str(
       input: input,
       format: Format.load_from_str(format),
       col_sep: col_sep,
+      convert_type: convert_type,
       &block
     )
   end
 
   # @param input [String] input string
-  # @param format [String] format string
+  # @param format [String, Format] format string
   # @param col_sep [String]
   #
   # @return [String]
-  def self.json(input:, format:, col_sep: DEFAULT_COL_SEP)
-    h = { 'vars' => parse_str(input: input, format: format, col_sep: col_sep) }
+  def self.json(input:,
+                format:,
+                convert_type: true,
+                col_sep: DEFAULT_COL_SEP)
+    h = {
+      'vars' => parse_str(
+        input: input,
+        format: format,
+        convert_type: convert_type,
+        col_sep: col_sep
+      )
+    }
+
     Oj.dump(h)
   end
 end

data/lib/csvpp/cli.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require 'gli'
+require_relative '../csvpp'
+require_relative '../csvpp/core_extensions'
+
+module CSVPP
+  # CSV++ command line interface.
+  #
+  # @example
+  #   exit CSVPP::CLI.run(ARGV)
+  #
+  class CLI
+    using CoreExtensions
+    extend GLI::App
+
+    CLIENT = FormatsClient.new
+
+    program_desc 'CSV++ Command Line Interface'
+
+    desc 'Parse input files'
+    command :parse do |c|
+      c.switch %i[convert-types], default_value: true, negatable: true
+      c.switch %i[open], default_value: true, negatable: true,
+        desc: 'Whether to open the output with the default application'
+
+      c.flag %i[f format], required: true,
+        desc: 'Format identifier or local file path to a JSON format'
+
+      c.flag %i[o output], required: true, desc: 'Output file'
+      c.flag %i[s separator], default_value: '|'
+      c.flag %i[open-cmd], default_value: OS.open_cmd
+
+      c.action do |global_options, options, args|
+        format = if File.exist?(options[:format])
+                   Format.load(options[:format])
+                 else
+                   CLIENT.format(options[:format])
+                 end
+
+        convert_type = options[:'convert-types']
+        output = options[:output].strip
+
+        case output
+        when /\.db$/
+          importer = SqliteImporter.new(
+            format: format,
+            db_path: output
+          )
+          importer.import(ARGF.read)
+        when /\.json$/
+          File.open(output, 'w') do |file|
+            file.puts CSVPP.json(
+              input: ARGF.read,
+              format: format,
+              convert_type: convert_type
+            )
+          end
+        end
+
+        OS.open(output, open_cmd: options[:'open-cmd']) if options[:open]
+      end
+    end
+
+    desc 'List formats'
+    command :formats do |c|
+      c.switch %i[web], default: false,
+        desc: 'Whether to view the formats in the web app'
+
+      c.action do |global_options, options, args|
+        if options[:web]
+          OS.open(CLIENT.base_uri)
+          next
+        end
+
+        puts CLIENT.formats.map(&:to_s)
+      end
+    end
+  end
+end

data/lib/csvpp/conversions.rb

@@ -4,19 +4,70 @@ module CSVPP
   module Conversions
     module_function
 
+    ARRAY_TYPE_RGX = /(?<array_type>\w+),\s*(?<array_delimiter>\W)/
+
     # @param obj [Object] object to parse
     # @param to [String] a type, e.g. "int"
-    # @missings [Array] list of values that are treated as missings, e.g. ['NA', '-', -999]
-    # @params options [Hash] options passed on to parsing methods for specific types
+    # @param missings [Array] list of values that are treated as missings,
+    #   e.g. ['NA', '-', -999]
+    # @param options [Hash] options passed on to parsing methods for specific types
     # @return parsed value, read from `obj`, interpreted as type given by `to`
     def convert(obj, to:, missings: [], **options)
       return nil if missing?(obj, missings)
 
+      if to.start_with?('array')
+        to, rest = to.split('<')
+        rest = rest.tr('>', '')
+        match = rest.match(ARRAY_TYPE_RGX)
+        options = options.merge(
+          type: match[:array_type],
+          delimiter: match[:array_delimiter]
+        )
+      end
+
       send("parse_#{to}", obj, **options)
     end
 
+    def parse_array(str, type:, delimiter:, **options)
+      str.split(delimiter).map { |entry| send("parse_#{type}", entry) }
+    end
+
+    def parse_chop(str, delimiter: ':', **options)
+      code, laterality, date = str.split(delimiter)
+      code = parse_string(code)
+      laterality = parse_string(laterality) if laterality
+      laterality = nil if laterality&.empty?
+      date = parse_date(date) if date
+
+      {
+        code: code,
+        laterality: laterality,
+        date: date
+      }
+    end
+
+    # See page 3 in documentation/Technisches_Begleitblatt_2017_d.pdf more more
+    # info on the medi data type.
+    def parse_medi(str, delimiter: ':', **options)
+      atc_code, annex, application, dose, unit = str.split(delimiter)
+      atc_code = parse_string(atc_code)
+      annex = parse_string(annex) if annex
+      annex = nil if annex&.empty?
+      application = parse_string(application)
+      dose = parse_decimal(dose)
+      unit = parse_string(unit)
+
+      {
+        atc_code: atc_code,
+        annex: annex,
+        application: application,
+        dose: dose,
+        unit: unit
+      }
+    end
+
     def parse_string(str, **options)
-      str.to_s
+      str.to_s.strip
     end
 
     def parse_int(str, **options)
@@ -40,7 +91,7 @@ module CSVPP
       Float(clean_decimal(str)) rescue nil
     end
 
-    def parse_decimal(str)
+    def parse_decimal(str, **options)
       return nil if str.to_s.empty?
 
       cleaned = clean_decimal(str).to_s
@@ -50,7 +101,6 @@ module CSVPP
       else
         BigDecimal(cleaned)
       end
-
     end
 
     def parse_date(str, **options)
@@ -61,7 +111,11 @@ module CSVPP
     # @param false_values [Array]: list of values that are interpreted as `false`
     # @return true or false, or
     #      nil if `str` doesn't match any value interpreted as `true` or `false`
-    def parse_boolean(str, true_values: [], false_values: [])
+    def parse_boolean(str,
+                      true_values: [],
+                      false_values: [],
+                      **options)
+
       cleaned = str.to_s.strip.downcase
 
       trues = if true_values.empty?

data/lib/csvpp/core_extensions.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module CSVPP
+  # Add core extensions here as refinements. Only load them when necessary, so
+  # that we don't monkeypatch applications that load CSV++.
+  #
+  # @example Loading the core extensions to the current lexical scope
+  #
+  #   module MyScope
+  #     using CSVPP::CoreExtensions
+  #
+  #     # Extensions exist here
+  #
+  #   end
+  #
+  #   # Extensions no longer exist here
+  #
+  module CoreExtensions
+    refine String do
+      def colorize(color_code)
+        "\e[#{color_code}m#{self}\e[0m"
+      end
+
+      def green
+        colorize 32
+      end
+
+      def blue
+        colorize 34
+      end
+
+      def pink
+        colorize 35
+      end
+
+      def yellow
+        colorize 33
+      end
+    end
+  end
+end

data/lib/csvpp/format.rb

@@ -1,44 +1,27 @@
 module CSVPP
   class Format
-    attr_reader :name, :skip
+    attr_reader :name, :description, :skip, :col_sep
 
-    class << self
-      # @param name [String] unique name of the format
-      # @param format [Format]
-      def add(name, format)
-        store[name] = format
-      end
-
-      # @param name [String] unique name of the format
-      def find(name)
-        store.fetch(name)
-      end
-
-      # @param path [String] path to format file
-      # @return [Format]
-      def load(path)
-        load_from_str File.read(path)
-      end
-
-      # @param json [String]
-      # @return [Format]
-      def load_from_str(json)
-        new Oj.load(json)
-      end
-
-      def all
-        store.values
-      end
+    # @param path [String] path to format file
+    # @return [Format]
+    def self.load(path)
+      return path if path.is_a? Format
+      load_from_str File.read(path)
+    end
 
-      def store
-        @store ||= {}
-      end
+    # @param json [String]
+    # @return [Format]
+    def self.load_from_str(json)
+      return json if json.is_a? Format
+      new Oj.load(json)
     end
 
     # @param format [Hash]
     def initialize(format)
       @name = format['name']
+      @description = format['description']
       @multiline = format['multiline'].to_s.strip.downcase == 'true'
+      @col_sep = format['column_separator']
       @skip = format['skip'].to_i
       @vars = format.fetch('vars')
 
@@ -51,6 +34,11 @@ module CSVPP
 
         @multiline_start = format.fetch('start')
       end
+
+      # Cache for actual indices because formats provide 1-based human readable
+      # positions. Only matters when parsing files with 30k+ line files. See
+      # #index(var).
+      @indices = {}
     end
 
     def var_names
@@ -62,7 +50,7 @@ module CSVPP
     end
 
     def index(var)
-      position(var) - 1
+      @indices[var] ||= position(var) - 1
     end
 
     def position(var)
@@ -87,7 +75,6 @@ module CSVPP
       array_from(var, 'true_values')
     end
 
-
     # Returns the values that are defined as `false` in the the format's json
     # definition for the given variable.
     # @return [Array] all values that should be interpreted as `false` for this variable
@@ -108,6 +95,10 @@ module CSVPP
       @multiline
     end
 
+    def to_s
+      "#{name.ljust(30)}\t| #{description}"
+    end
+
     private
 
     # Returns the value or values specified for the given attribute of the given

data/lib/csvpp/formats_client.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'httparty'
+
+module CSVPP
+  class FormatsClient
+    include HTTParty
+
+    DEFAULT_HOST = 'http://formats.iapps.swissdrg.local'
+
+    def initialize(host: DEFAULT_HOST)
+      self.class.base_uri ENV['FORMATS_HOST'] || host
+    end
+
+    # @return [String] e.g. "http://formats.iapps.swissdrg.local"
+    def base_uri
+      self.class.base_uri
+    end
+
+    # @return [Array<Format>]
+    def formats
+      self.class.get('/api/formats').map { |hash| Format.new(hash) }
+    end
+
+    # @param name [String]
+    # @return [Format]
+    def format(name)
+      response = self.class.get("/api/formats/#{name}")
+
+      if (error = response['error'])
+        raise ArgumentError, %{#{error} "#{name}"}
+      end
+
+      Format.new(response)
+    end
+  end
+end

data/lib/csvpp/os.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module CSVPP
+  # Provides utility functions for determining OS and OS-specific system calls.
+  module OS
+    module_function
+
+    # http://stackoverflow.com/a/171011/1314848.
+    def windows?
+      !!(/cygwin|mswin|mingw|bccwin|wince|emx/ =~ ruby_platform)
+    end
+
+    def unix?
+      !windows?
+    end
+
+    def mac?
+      !!(/darwin/ =~ ruby_platform)
+    end
+
+    def linux?
+      unix? && !mac?
+    end
+
+    def open(str, open_cmd: self.open_cmd)
+      system "#{open_cmd} #{str}"
+    end
+
+    def open_cmd
+      if mac?
+        'open'
+      elsif linux?
+        'xdg-open'
+      elsif windows?
+        'START ""'
+      else
+        raise 'Unsupported OS'
+      end
+    end
+
+    def pager
+      return nil if windows?
+      ENV['PAGER'] || 'less'
+    end
+
+    def ruby_platform
+      RUBY_PLATFORM
+    end
+  end
+end

data/lib/csvpp/parser.rb

@@ -18,7 +18,7 @@ module CSVPP
       new(
         format: format,
         col_sep: col_sep,
-        convert_type: convert_type,
+        convert_type: convert_type
       ).parse(input, &block)
     end
 
@@ -28,28 +28,24 @@ module CSVPP
     #
     # @return [Array<Object>]
     def self.parse_str(input:,
-                   format:,
-                   col_sep: DEFAULT_COL_SEP,
-                   convert_type: true,
-                   &block)
+                       format:,
+                       col_sep: DEFAULT_COL_SEP,
+                       convert_type: true,
+                       &block)
 
       new(
         format: format,
         col_sep: col_sep,
-        convert_type: convert_type,
+        convert_type: convert_type
       ).parse_str(input, &block)
     end
 
     def initialize(format:, col_sep: DEFAULT_COL_SEP, convert_type: true)
       @format = format
-      @col_sep = col_sep
+      @col_sep = format.col_sep || col_sep
       @convert_type = convert_type
     end
 
-    def convert_type?
-      !!@convert_type
-    end
-
     def parse(path, &block)
       parse_io(File.open(path), &block)
     end
@@ -66,25 +62,26 @@ module CSVPP
 
     def set_value!(hash, var, value)
       hash[var] = value
-
-      if convert_type?
-        type = format.type(var)
-        return if type.nil?
-
-        hash[var] = convert(value,
-                            to: type,
-                            missings: format.missings(var),
-                            true_values: format.true_values(var),
-                            false_values: format.false_values(var))
-      end
+      return unless @convert_type
+
+      type = format.type(var)
+      return if type.nil?
+
+      hash[var] = convert(
+        value,
+        to: type,
+        missings: format.missings(var),
+        true_values: format.true_values(var),
+        false_values: format.false_values(var)
+      )
     end
 
-    def add_result!(results, hash, &block)
-      if block_given? && (obj = block.call(hash))
-        results << obj
-      else
-        results << hash
-      end
+    def add_result!(results, hash)
+      results << if block_given? && (obj = yield(hash))
+                   obj
+                 else
+                   hash
+                 end
     end
 
     def parse_io(io, &block)
@@ -98,10 +95,11 @@ module CSVPP
 
         hash = {}
         format.var_names.each do |var|
-          hash["line_number"] = line_number
+          hash['line_number'] = line_number
 
           index = format.index(var)
-          value = columns[index].strip
+          value = columns[index]
+
           set_value!(hash, var, value)
         end
 
@@ -127,14 +125,14 @@ module CSVPP
 
           # ...and start building a new one.
           hash = {}
-          hash["line_number"] = line_number
+          hash['line_number'] = line_number
         end
 
         next if hash.nil?
 
         format.vars_for_line(line_id).each do |var|
           index = format.index(var)
-          value = columns[index].strip
+          value = columns[index]
           set_value!(hash, var, value)
         end
       end
@@ -157,6 +155,5 @@ module CSVPP
         yield(line, index) unless index < offset
       end
     end
-
   end
 end

data/lib/csvpp/sqlite_importer.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'sequel'
+
+module CSVPP
+  # Imports data into an Sqlite database.
+  class SqliteImporter
+    attr_reader :format, :data, :db
+
+    # @param format [Format]
+    # @param db_path [String]
+    def initialize(format:, db_path:)
+      @format = format
+      @db = Sequel.sqlite(db_path)
+
+      @db.drop_table? :data
+      @db.create_table :data do
+        Int :line_number
+
+        format.var_names.each do |var|
+          type = format.type(var)
+
+          if type.start_with?('array')
+            type = 'String'
+          else
+            type.capitalize
+          end
+
+          send(type, var)
+        end
+      end
+
+      @data = @db[:data]
+    end
+
+    # @param input [String] input string
+    def import(input)
+      Parser.parse_str(
+        input: input,
+        format: format,
+        convert_type: false
+      ) do |attr|
+        data.insert(attr)
+      end
+    end
+  end
+end

data/lib/csvpp/version.rb

@@ -1,3 +1,3 @@
 module CSVPP
-  VERSION = "0.3.1"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csvpp
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - SwissDRG AG
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-03-22 00:00:00.000000000 Z
+date: 2018-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: oj
@@ -16,14 +16,76 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.3.9
+        version: 3.6.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.3.9
+        version: 3.6.2
+- !ruby/object:Gem::Dependency
+  name: gli
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.17.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.17.1
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.16.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.16.2
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.13
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.13
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.9'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -108,6 +170,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-prof
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: CSV++
 email:
 - rathesan.iyadurai@swissdrg.org
@@ -118,6 +208,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".gitlab-ci.yml"
+- ".rubocop.yml"
 - ".ruby-version"
 - ".travis.yml"
 - Gemfile
@@ -126,13 +217,20 @@ files:
 - README.md
 - Rakefile
 - bin/console
+- bin/profile
 - bin/setup
 - csvpp.gemspec
+- documentation/Technisches_Begleitblatt_2017_d.pdf
 - exe/csvpp
 - lib/csvpp.rb
+- lib/csvpp/cli.rb
 - lib/csvpp/conversions.rb
+- lib/csvpp/core_extensions.rb
 - lib/csvpp/format.rb
+- lib/csvpp/formats_client.rb
+- lib/csvpp/os.rb
 - lib/csvpp/parser.rb
+- lib/csvpp/sqlite_importer.rb
 - lib/csvpp/version.rb
 homepage: https://www.swissdrg.org
 licenses:
@@ -154,7 +252,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: CSV++