csv2schema 0.1.0 → 0.1.1

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NTA1OWRiNjIxMmQ2YjdiZmMxM2M0MzkzOTI5OWNiNjE5Y2NmNGQzZA==
+    NTRjMDQyMjU2NmY1YTYzOTc4MGMzMjkxMjE4MDk3MjUyYjQzMThmZA==
   data.tar.gz: !binary |-
-    Nzk0MGUxODRlMTVlZmY4Y2VlNDY2ZmQ2MGQ3ZGI1ZWY4ZDQzMjEyOA==
+    OWMwYjAyNzUwMDAwNjRmZjNhZTAxNDMwMjgyMjhmNjVlZGU2ZWQ5ZA==
 SHA512:
   metadata.gz: !binary |-
-    NjE1YjMzZDIxZGQyMDllZjMyNWU2MDBhZmVmMmZjMDE3ODA0YmMzMDk1YzY5
-    MGQ5YzFiOWEwOWU3MTYwYTJkYzdlN2YzYWE4OTY1MjFhZGI0MDdlMGNmYTNh
-    Mzk1MGU2N2EyMDNmYjZlMDVhMGE1MTg2N2IwYzJkNTVkMmNmYWI=
+    Y2VlOWMzZGM1OWQyYjA3YzdhMzM5MDBmNjcwNmZkZjFmNzk0MTFhZDhmNzll
+    ZTdjYmE3ZTQ4NDBhNDkzMDFkYjY1NzgyYTgwNWE2NjgwM2YxN2NmYzM1MTM5
+    MTg0MjM5OTcxMTk3OTg1MTdhZTFjYjVjNzNkYjU3NWI5ZmZiZWY=
   data.tar.gz: !binary |-
-    NmUwNTQ5ZTU0MjNhNTY5MmQ4ZWNmMWUyNGFkZDk1M2M2YTYyYzgxZGVhNGI0
-    ZTViNzY3NGZmMzA2NmI3ZTBkNGVkNDc1MjQ5ZTJkMTU4NDZhNDJhOGQzYzU1
-    N2RhZmIzZjdlNWJlY2YzMWUzNDM0MWI2NzcxMTdmMDEzMDdmM2E=
+    YzA3MzRhYjY0ODJkOTRmMWZjOTQ5NzhkYjY3NTA5NzFmNzVjOWIyYWJjOThi
+    ZDcxMmQ3MWExOGE4ZmM1YzQyMWUxYTBiYWExYTMzOTVmOWMxYmZlOGMzNDZi
+    OGY0ZWUyN2ViZGVkZDllZDk1NzRjZGM5YWJkZTg4NWFmYjM1Zjc=

data/.travis.yml

@@ -1,3 +1,4 @@
+---
 language: ruby
 rvm:
 - 2.3.1
@@ -10,3 +11,8 @@ deploy:
   on:
     tags: true
     repo: theodi/csv2schema
+env:
+  global:
+    secure: jNfyOFHHWsNoOnpZeeB64rfpgBy1qn4h7pltuSwvW5LwuHBaQQU2d2vqi7ypqN7AfCvxU2ZGOmiGlhX7TjsIFhqs0LBpUuzde7xgMMrHuDr7fp3aNfKXjljdg0ltUsXjcgyoPMsaWJt9KqhIY3YFGcz+PeOFGmUVLDHUNjM2nyDIlmPxn5iCt/UzSOXWW3iHdjwfJyiUNLvRP+AxCfL0bR7d0LleitAk75ITcsKgbasekg51uFC/wvdZnPB88Er3Sr4y2/YXQDjQuyCj+cOZitNgHjFQ3lRz7CBHko5uA7NcQeFxLmV3oaOTmbwDGiI9fHjzWXDkWAc2jnf98B1UizFJDkGpftXBRrtoFqf6huIegJf+0bP4zKoEOfINHm50QGVLmbXDqN0Oc3WNJo/ULXm80W28yf7cuQBHSJ8ze+3Bllg8NHQwlCws2teHappkOG842Nzzx9qNs/ATleiHtvIo6iUtv5QXHkqD7OET6ubJVyGWw5guCRo/tpsFV1djEc3O9Dpc2aQs2hhz0gMvHVrb1V1mGGKwC37KEJCzRuPJypkTvSsylTLezq4eQ24E/jMpVMi97vceXbXpj+mxL1aaOwtDoN5QR6MCvC3w7dvODEG/mJ5wDtEhPp35zK3iwI26g/gxWrW9iy9cLz2jqBPY1EZJDOPvlD4OyB+XNSc=
+after_success:
+- bundle exec henry deploy

data/CHANGELOG.md

@@ -0,0 +1,9 @@
+# Change Log
+
+## [0.1.0](https://github.com/theodi/csv2schema/tree/0.1.0) (2016-09-02)
+[Full Changelog](https://github.com/theodi/csv2schema/compare/0.0.1...0.1.0)
+
+## [0.0.1](https://github.com/theodi/csv2schema/tree/0.0.1) (2016-09-02)
+
+
+\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)*

data/README.md

@@ -26,6 +26,32 @@ Or install it yourself as:
 
 ## Usage
 
+You can either use the gem on the command line, or in your own Ruby code
+
+### On the command line
+
+After installing the gem, you can generate a schema file on the command line like so:
+
+```
+csv2schema generate myfile.csv
+```
+
+This will generate a JSON table schema file at `schema.json`.
+
+You can also specify a filename and extension like so:
+
+```
+csv2schema generate myfile.csv --filename=myschema.json
+```
+
+If you want to dump the JSON to the command line (to pipe somewhere else for example) we can handle that too:
+
+```
+csv2schema generate myfile.csv --dump
+```
+
+### In your own Ruby code
+
 Require the gem (if you haven't already):
 
 ```ruby

data/csv2schema.gemspec

@@ -27,9 +27,12 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "csvlint", "~> 0.3.2"
+  spec.add_dependency "thor", "~> 0.19.1"
+  spec.add_dependency "colorize", "~> 0.8.1"
 
   spec.add_development_dependency "bundler", "~> 1.11"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "pry", "~> 0.10.4"
+  spec.add_development_dependency "henry", "~> 0.1.3"
 end

data/exe/csv2schema

@@ -0,0 +1,51 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "csv2schema"
+require "thor"
+require "colorize"
+
+module Csv2schema
+  class Cli < Thor
+
+    desc "csv2schema /path/to/file.csv OR csv2schema http://example.org/file.csv", "Generate schema from CSV"
+    option :dump, type: :boolean
+    option :filename
+    def generate(source)
+      source = read_source(source)
+      csv = Csv2schema::CSV.new(source)
+      if options[:dump]
+        puts csv.schema
+      else
+        filename = options[:filename].nil? ? 'schema.json' : options[:filename]
+        File.open(filename, 'w') {|f| f.write(csv.schema) }
+        puts "Schema created at #{filename}!".colorize(:green)
+      end
+    end
+
+    private
+
+      def read_source(source)
+        if source.nil?
+          # If no source is present, try reading from stdin
+          if !$stdin.tty?
+            source = StringIO.new(STDIN.read) rescue nil
+            return_error "No CSV data to validate" if !options[:schema] && source.nil?
+          end
+        else
+          # If the source isn't a URL, it's a file
+          unless source =~ /^http(s)?/
+            begin
+              source = File.new( source )
+            rescue Errno::ENOENT
+              return_error "#{source} not found"
+            end
+          end
+        end
+        source
+      end
+
+  end
+end
+
+Csv2schema::Cli.start(ARGV)

data/lib/csv2schema.rb

@@ -3,6 +3,7 @@ require "json"
 
 require "csv2schema/version"
 
+require "csv2schema/format_builder"
 require "csv2schema/formats"
 require "csv2schema/required"
 require "csv2schema/schema"

data/lib/csv2schema/csv.rb

@@ -5,7 +5,7 @@ module Csv2schema
     include Csv2schema::Schema
 
     def initialize(io)
-      @validator = Csvlint::Validator.new(io)
+      @validator = Csv2schema::Validator.new(io)
       @column_headers = get_column_headers
       apply_formats
       apply_required

data/lib/csv2schema/format_builder.rb

@@ -0,0 +1,84 @@
+module Csv2schema
+  class Validator < Csvlint::Validator
+
+      def build_formats(row)
+        row.each_with_index do |col, i|
+          next if col.nil? || col.empty?
+          @formats[i] ||= Hash.new(0)
+
+        #  require "pry" ; binding.pry
+
+          format =
+              if float?(col)
+                :float
+              elsif col.strip[FORMATS[:boolean]]
+                :boolean
+              elsif uri?(col)
+                :uri
+              elsif possible_date?(col)
+                date_formats(col)
+              elsif col.strip[FORMATS[:numeric]]
+                :numeric
+              else
+                :string
+              end
+
+          @formats[i][format] += 1
+        end
+      end
+
+      def float?(col)
+        Float(col).to_s == col rescue false
+      end
+
+      def date_formats(col)
+        if col[FORMATS[:date_db]] && date_format?(Date, col, '%Y-%m-%d')
+          :date_db
+        elsif col[FORMATS[:date_short]] && date_format?(Date, col, '%e %b')
+          :date_short
+        elsif col[FORMATS[:date_rfc822]] && date_format?(Date, col, '%e %b %Y')
+          :date_rfc822
+        elsif col[FORMATS[:date_long]] && date_format?(Date, col, '%B %e, %Y')
+          :date_long
+        elsif col[FORMATS[:dateTime_time]] && date_format?(Time, col, '%H:%M')
+          :dateTime_time
+        elsif col[FORMATS[:dateTime_hms]] && date_format?(Time, col, '%H:%M:%S')
+          :dateTime_hms
+        elsif col[FORMATS[:dateTime_db]] && date_format?(Time, col, '%Y-%m-%d %H:%M:%S')
+          :dateTime_db
+        elsif col[FORMATS[:dateTime_iso8601]] && date_format?(Time, col, '%Y-%m-%dT%H:%M:%SZ')
+          :dateTime_iso8601
+        elsif col[FORMATS[:dateTime_short]] && date_format?(Time, col, '%d %b %H:%M')
+          :dateTime_short
+        elsif col[FORMATS[:dateTime_long]] && date_format?(Time, col, '%B %d, %Y %H:%M')
+          :dateTime_long
+        elsif col[FORMATS[:date_month]] && date_format?(Date, col, '%Y-%m')
+          :date_month
+        elsif col.strip[FORMATS[:numeric]]
+          :numeric
+        else
+          :string
+        end
+      end
+
+      FORMATS = {
+        :string => nil,
+        :numeric => /\A[-+]?\d*\.?\d+(?:[eE][-+]?\d+)?\z/,
+        :uri => /\Ahttps?:/,
+        :boolean => /\Atrue\z|\Afalse\z|\Ay\z|\An\z|\Ayes\z|\Ano\z|\A1\z|\A0\z/i,
+        :date_year => /\A\d{4,}\z/,                                                       # 12345
+        :date_month => /\A\d{4,}-\d{2,}\z/,                                               # 12345-02
+        :date_db => /\A\d{4,}-\d\d-\d\d\z/,                                               # "12345-01-01"
+        :date_long => /\A(?:#{Date::MONTHNAMES.join('|')}) [ \d]\d, \d{4,}\z/,            # "January  1, 12345"
+        :date_rfc822 => /\A[ \d]\d (?:#{Date::ABBR_MONTHNAMES.join('|')}) \d{4,}\z/,      # " 1 Jan 12345"
+        :date_short => /\A[ \d]\d (?:#{Date::ABBR_MONTHNAMES.join('|')})\z/,              # "1 Jan"
+        :dateTime_db => /\A\d{4,}-\d\d-\d\d \d\d:\d\d:\d\d\z/,                            # "12345-01-01 00:00:00"
+        :dateTime_hms => /\A\d\d:\d\d:\d\d\z/,                                            # "00:00:00"
+        :dateTime_iso8601 => /\A\d{4,}-\d\d-\d\dT\d\d:\d\d:\d\dZ\z/,                      # "12345-01-01T00:00:00Z"
+        :dateTime_long => /\A(?:#{Date::MONTHNAMES.join('|')}) \d\d, \d{4,} \d\d:\d\d\z/, # "January 01, 12345 00:00"
+        :dateTime_short => /\A\d\d (?:#{Date::ABBR_MONTHNAMES.join('|')}) \d\d:\d\d\z/,   # "01 Jan 00:00"
+        :dateTime_time => /\A\d\d:\d\d\z/,                                                # "00:00"
+      }.freeze
+
+  end
+end

data/lib/csv2schema/schema.rb

@@ -21,7 +21,14 @@ module Csv2schema
     def get_type(format)
       {
         string: 'http://www.w3.org/2001/XMLSchema#string',
-        numeric: 'http://www.w3.org/2001/XMLSchema#int'
+        numeric: 'http://www.w3.org/2001/XMLSchema#int',
+        float: 'http://www.w3.org/2001/XMLSchema#float',
+        uri: 'http://www.w3.org/2001/XMLSchema#anyURI',
+        boolean: 'http://www.w3.org/2001/XMLSchema#boolean',
+        date_db: 'http://www.w3.org/2001/XMLSchema#date',
+        dateTime_iso8601: 'http://www.w3.org/2001/XMLSchema#dateTime',
+        date_month: 'http://www.w3.org/2001/XMLSchema#gYearMonth',
+        dateTime_hms: 'http://www.w3.org/2001/XMLSchema#time'
       }[format]
     end
 

data/lib/csv2schema/version.rb

@@ -1,3 +1,3 @@
 module Csv2schema
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: csv2schema
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - pezholio
@@ -24,6 +24,34 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.3.2
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -80,10 +108,25 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.10.4
+- !ruby/object:Gem::Dependency
+  name: henry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.3
 description: 
 email:
 - pezholio@gmail.com
-executables: []
+executables:
+- csv2schema
 extensions: []
 extra_rdoc_files: []
 files:
@@ -91,6 +134,7 @@ files:
 - .rspec
 - .ruby-version
 - .travis.yml
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
@@ -99,8 +143,10 @@ files:
 - bin/console
 - bin/setup
 - csv2schema.gemspec
+- exe/csv2schema
 - lib/csv2schema.rb
 - lib/csv2schema/csv.rb
+- lib/csv2schema/format_builder.rb
 - lib/csv2schema/formats.rb
 - lib/csv2schema/required.rb
 - lib/csv2schema/schema.rb