csv-import-analyzer 0.0.1

data/spec/csv-import-analyzer/helpers/string_class_extension_spec.rb

@@ -0,0 +1,18 @@
+# require 'spec_helper'
+
+describe 'substr_count' do
+  context 'different possible delimiters' do
+    it 'returns count of commas as delimiter in a string' do
+      expect("hello, hi, how, are you?".substr_count(",")).to eq(3)
+    end
+    it 'returns count of semi-colons as delimiter in a string' do
+      expect("hello; hi, how, are you?".substr_count(";")).to eq(1)
+    end
+    it 'returns count of pipe as delimiter in a string' do
+      expect("hello, hi| how| are you?".substr_count("|")).to eq(2)
+    end
+    it 'returns count of tab as delimiter in a string' do
+      expect("hello\thi\thow| are you?".substr_count("\t")).to eq(2)
+    end
+  end
+end

data/spec/csv-import-analyzer/query_builder/mysql_query_helper_spec.rb

@@ -0,0 +1,54 @@
+# require 'spec_helper'
+class DummyClass
+end
+describe '#form_query_for_datatype' do
+  before(:each) do
+    @dummy_class = DummyClass.new
+    @dummy_class.extend(CsvImportAnalyzer::MysqlQueryHelper)
+  end
+  context 'expected arguments are not set' do
+    let(:args) {Hash[:header => :test]}
+    let(:args1) {Hash[:datatype, :test]}
+    it ' returns missing arguments error' do
+      expect(@dummy_class.form_query_for_datatype(args)).to be_instance_of(MissingRequiredArguments)
+    end
+    it 'returns invalid if set to nil' do
+      expect(@dummy_class.form_query_for_datatype(args1)).to be_instance_of(MissingRequiredArguments)
+    end
+  end
+
+  context 'expected arguments are set' do
+    let(:args) {Hash[:header => :test, :datatype => :string]}
+    let(:args1) {Hash[:header => :test, :datatype => :integer]}
+    it 'returns expected sql query for string' do
+      expect(@dummy_class.form_query_for_datatype(args)).to eq("test varchar(255)")
+    end
+    it 'returns expected sql query for numeric' do
+      expect(@dummy_class.form_query_for_datatype(args1)).to eq("test integer")
+    end
+  end
+
+end
+describe '#import_csv' do
+  before(:each) do
+    @dummy_class = DummyClass.new
+    @dummy_class.extend(CsvImportAnalyzer::MysqlQueryHelper)
+  end
+  context 'expected arguments are not set' do
+    let(:args) {Hash[:tablename => "test", :delimiter => ","]}
+    let(:args1) {Hash[:filename => "test"]}
+    it ' return SqlQueryErrror' do
+      expect(@dummy_class.import_csv(args)).to be_instance_of(MissingRequiredArguments)
+    end
+    it 'should return SqlQueryErrror' do
+      expect(@dummy_class.import_csv(args1)).to be_instance_of(MissingRequiredArguments)
+    end
+  end
+
+  context 'expected arguments are set' do
+    let(:args) {Hash[:tablename => "test", :delimiter => ",", :filename => "sample.csv"]}
+    it 'returns expected import query' do
+      expect(@dummy_class.import_csv(args)).to eq("LOAD DATA INFILE sample.csv INTO TABLE test FIELDS TERMINATED BY ',' ENCLOSED BY '\"' LINES TERMINATED BY '\\n' IGNORE 1 LINES;")
+    end
+  end
+end

data/spec/csv-import-analyzer/query_builder/pg_query_helper_spec.rb

@@ -0,0 +1,55 @@
+# require 'spec_helper'
+class DummyClass
+end
+describe '#form_query_for_datatype' do
+  before(:each) do
+    @dummy_class = DummyClass.new
+    @dummy_class.extend(CsvImportAnalyzer::PgQueryHelper)
+  end
+  context 'expected arguments are not set' do
+    let(:args) {Hash[:header => :test]}
+    let(:args1) {Hash[:datatype, :test]}
+    it ' returns missing arguments error' do
+      expect(@dummy_class.form_query_for_datatype(args)).to be_instance_of(MissingRequiredArguments)
+    end
+    it 'returns invalid if set to nil' do
+      expect(@dummy_class.form_query_for_datatype(args1)).to be_instance_of(MissingRequiredArguments)
+    end
+  end
+
+  context 'expected arguments are set' do
+    let(:args) {Hash[:header => :test, :datatype => :string]}
+    let(:args1) {Hash[:header => :test, :datatype => :integer]}
+    it 'returns expected sql query for string' do
+      expect(@dummy_class.form_query_for_datatype(args)).to eq("test varchar(255)")
+    end
+    it 'returns expected sql query for numeric' do
+      expect(@dummy_class.form_query_for_datatype(args1)).to eq("test integer")
+    end
+  end
+
+end
+describe '#import_csv' do
+  before(:each) do
+    @dummy_class = DummyClass.new
+    @dummy_class.extend(CsvImportAnalyzer::PgQueryHelper)
+  end
+  context 'expected arguments are not set' do
+    let(:args) {Hash[:tablename => "test", :delimiter => ","]}
+    let(:args1) {Hash[:filename => "test"]}
+    it ' return SqlQueryErrror' do
+      expect(@dummy_class.import_csv(args)).to be_instance_of(MissingRequiredArguments)
+    end
+    it 'should return SqlQueryErrror' do
+      expect(@dummy_class.import_csv(args1)).to be_instance_of(MissingRequiredArguments)
+    end
+  end
+
+  context 'expected arguments are set' do
+    let(:args) {Hash[:tablename => "test", :delimiter => ",", :filename => "filename"]}
+    it 'returns expected import query' do
+      expect(@dummy_class.import_csv(args)).to eq("COPY test FROM 'filename' HEADER DELIMITER ',' CSV NULL AS 'NULL';")
+    end
+  end
+
+end

data/spec/csv-import-analyzer_spec.rb

@@ -0,0 +1,14 @@
+# require 'spec_helper'
+
+# CsvImportAnalyzer.process("sampleTab.csv", {:metadata_output => true})
+
+
+describe CsvImportAnalyzer do
+  include CsvImportAnalyzer
+  it 'should return invalid file as file not found' do
+    expect(CsvImportAnalyzer.process("sample.csv")).to be_instance_of(FileNotFound)
+  end
+  it 'should be able to process a valid file' do
+    expect(CsvImportAnalyzer.process($sample_csv_path)).not_to be_instance_of(FileNotFound)
+  end
+end

data/spec/fixtures/sample.csv

@@ -0,0 +1,5 @@
+Year ID,Make ID,Model ID,Description ID,Price ID
+1997,Ford,,"ac, abs, moon","3000"
+1999,Chevy,"Venture ""Extended Edition""","",4900.00
+1999,'Chevy',"Venture ""Extended Edition, Very Large""",,5000.00
+1996,Jeep,Grand Che'rokee,"MUST SELL!air, moon roof, loaded",4799.00

data/spec/fixtures/sample_options.yml

@@ -0,0 +1,11 @@
+preferences:
+  metadata_output: nil
+  processed_input: nil
+  unique: 10
+  check_bounds: true
+  datatype_analysis: 200
+  chunk: 20
+  database: [pg, mysql]
+  quote_convert: true
+  replace_nulls: true
+  out_format: :json

data/spec/fixtures/semicolon-sample.csv

@@ -0,0 +1,5 @@
+Year ID;Make ID;Model ID;Description ID;Price ID
+1997;Ford;;"ac, abs; moon";"3000"
+1999;Chevy;"Venture ""Extended Edition""";"";4900.00
+1999;'Chevy';"Venture ""Extended Edition; Very Large""";;5000.00
+1996;Jeep;Grand Che'rokee;"MUST SELL!air; moon roof; loaded";4799.00

data/spec/spec_helper.rb

@@ -0,0 +1,84 @@
+require 'simplecov'
+SimpleCov.start
+require 'csv-import-analyzer'
+$sample_csv_path = "/home/avinash/Desktop/csv-import-analyzer/spec/fixtures/sample.csv"
+$sample_ssv_path = "/home/avinash/Desktop/csv-import-analyzer/spec/fixtures/semicolon-sample.csv"
+
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause this
+# file to always be loaded, without a need to explicitly require it in any files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, make a
+# separate helper file that requires this one and then use it only in the specs
+# that actually need it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # These two settings work together to allow you to limit a spec run
+  # to individual examples or groups you care about by tagging them with
+  # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  # get run.
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # Enable only the newer, non-monkey-patching expect syntax.
+    # For more details, see:
+    #   - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
+    expectations.syntax = :expect
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Enable only the newer, non-monkey-patching expect syntax.
+    # For more details, see:
+    #   - http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+    mocks.syntax = :expect
+
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended.
+    mocks.verify_partial_doubles = true
+  end
+=end
+end

metadata

@@ -0,0 +1,208 @@
+--- !ruby/object:Gem::Specification
+name: csv-import-analyzer
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- avinash vallabhaneni
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: smarter_csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.17
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.17
+- !ruby/object:Gem::Dependency
+  name: roo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+description: Santize large csv files and help in predicting datatypes including min
+  max values for easy import to SQL
+email:
+- avinash.vallab@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- csv-import-analyzer.gemspec
+- lib/csv-import-analyzer.rb
+- lib/csv-import-analyzer/analyzer/csv_check_bounds.rb
+- lib/csv-import-analyzer/analyzer/delimiter_identifier.rb
+- lib/csv-import-analyzer/analyzer/file_type_assertion.rb
+- lib/csv-import-analyzer/csv_datatype_analysis.rb
+- lib/csv-import-analyzer/csv_sanitizer.rb
+- lib/csv-import-analyzer/export/metadata_analysis.rb
+- lib/csv-import-analyzer/helpers/common_functions.rb
+- lib/csv-import-analyzer/helpers/datatype_validation.rb
+- lib/csv-import-analyzer/helpers/errors.rb
+- lib/csv-import-analyzer/helpers/string_class_extensions.rb
+- lib/csv-import-analyzer/query_builder/mysql_query_helper.rb
+- lib/csv-import-analyzer/query_builder/pg_query_helper.rb
+- lib/csv-import-analyzer/query_builder/query_helper.rb
+- lib/csv-import-analyzer/sampleTab.csv
+- lib/csv-import-analyzer/sql_query_builder.rb
+- lib/csv-import-analyzer/version.rb
+- lib/metadata_output.json
+- lib/sampleTab.csv
+- spec/csv-import-analyzer/analyzer/csv_check_bounds_spec.rb
+- spec/csv-import-analyzer/analyzer/delimiter_identifier_spec.rb
+- spec/csv-import-analyzer/analyzer/file_type_assertion_spec.rb
+- spec/csv-import-analyzer/csv_datatype_analysis_spec.rb
+- spec/csv-import-analyzer/csv_sanitizer_spec.rb
+- spec/csv-import-analyzer/export/metadata_analysis_spec.rb
+- spec/csv-import-analyzer/helpers/common_functions_spec.rb
+- spec/csv-import-analyzer/helpers/csv_check_bounds_spec.rb
+- spec/csv-import-analyzer/helpers/datatype_validation_spec.rb
+- spec/csv-import-analyzer/helpers/mysql_query_helper_spec.rb
+- spec/csv-import-analyzer/helpers/pq_query_helper_spec.rb
+- spec/csv-import-analyzer/helpers/string_class_extension_spec.rb
+- spec/csv-import-analyzer/query_builder/mysql_query_helper_spec.rb
+- spec/csv-import-analyzer/query_builder/pg_query_helper_spec.rb
+- spec/csv-import-analyzer_spec.rb
+- spec/fixtures/sample.csv
+- spec/fixtures/sample_options.yml
+- spec/fixtures/semicolon-sample.csv
+- spec/spec_helper.rb
+homepage: http://rubygems.org/gems/csv-import-analyzer
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: To process large csv files and predict valid datatypes of each column for
+  easy import into SQL
+test_files:
+- spec/csv-import-analyzer/analyzer/csv_check_bounds_spec.rb
+- spec/csv-import-analyzer/analyzer/delimiter_identifier_spec.rb
+- spec/csv-import-analyzer/analyzer/file_type_assertion_spec.rb
+- spec/csv-import-analyzer/csv_datatype_analysis_spec.rb
+- spec/csv-import-analyzer/csv_sanitizer_spec.rb
+- spec/csv-import-analyzer/export/metadata_analysis_spec.rb
+- spec/csv-import-analyzer/helpers/common_functions_spec.rb
+- spec/csv-import-analyzer/helpers/csv_check_bounds_spec.rb
+- spec/csv-import-analyzer/helpers/datatype_validation_spec.rb
+- spec/csv-import-analyzer/helpers/mysql_query_helper_spec.rb
+- spec/csv-import-analyzer/helpers/pq_query_helper_spec.rb
+- spec/csv-import-analyzer/helpers/string_class_extension_spec.rb
+- spec/csv-import-analyzer/query_builder/mysql_query_helper_spec.rb
+- spec/csv-import-analyzer/query_builder/pg_query_helper_spec.rb
+- spec/csv-import-analyzer_spec.rb
+- spec/fixtures/sample.csv
+- spec/fixtures/sample_options.yml
+- spec/fixtures/semicolon-sample.csv
+- spec/spec_helper.rb