csvsql 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 77f9bf18a9ec8773b4015b456cd4dc789bebd36d
+  data.tar.gz: 6c18bed0e815307ffa18cd8b9ef80a774abecaa9
+SHA512:
+  metadata.gz: 567347889397e591db8a04dc8dd9b8154e4915eb85b88575f62767a7e78bc578e2282d8250cd27f5947cc7462e3bd37f6869c455ec76ada203be4c8e03b1da7b
+  data.tar.gz: 4c345a17fb5b380d6039fbb1bfad0609596a1703c8dca07b65cd917f5bfa0b77d49e7d3b41181d6f6cf7195b8ebc8f5d869099aebfc0d2b8dc190de64e303656

data/.gitignore

@@ -0,0 +1,13 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+*.swp

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require gem_helper

data/.rubocop.yml

@@ -0,0 +1,137 @@
+# https://github.com/rails/rails/blob/master/.rubocop.yml
+#
+AllCops:
+  TargetRubyVersion: 2.4
+  # RuboCop has a bunch of cops enabled by default. This setting tells RuboCop
+  # to ignore them, so only the ones explicitly set in this file are enabled.
+  DisabledByDefault: true
+
+# Prefer &&/|| over and/or.
+Style/AndOr:
+  Enabled: true
+
+# Do not use braces for hash literals when they are the last argument of a
+# method call.
+Style/BracesAroundHashParameters:
+  Enabled: true
+
+# Align `when` with `case`.
+Layout/CaseIndentation:
+  Enabled: true
+  EnforcedStyle: end
+  IndentOneStep: false
+
+# Align comments with method definitions.
+Layout/CommentIndentation:
+  Enabled: true
+
+Layout/EmptyLineAfterMagicComment:
+  Enabled: true
+
+# No extra empty lines.
+Layout/EmptyLines:
+  Enabled: true
+
+# In a regular class definition, no empty lines around the body.
+Layout/EmptyLinesAroundClassBody:
+  Enabled: true
+
+# In a regular method definition, no empty lines around the body.
+Layout/EmptyLinesAroundMethodBody:
+  Enabled: true
+
+# In a regular module definition, no empty lines around the body.
+Layout/EmptyLinesAroundModuleBody:
+  Enabled: true
+
+# Use Ruby >= 1.9 syntax for hashes. Prefer { a: :b } over { :a => :b }.
+Style/HashSyntax:
+  Enabled: true
+
+# Method definitions after `private` or `protected` isolated calls need one
+# extra level of indentation.
+Layout/IndentationConsistency:
+  Enabled: true
+  # EnforcedStyle: rails
+
+# Two spaces, no tabs (for indentation).
+Layout/IndentationWidth:
+  Enabled: true
+
+Layout/SpaceAfterColon:
+  Enabled: true
+
+Layout/SpaceAfterComma:
+  Enabled: true
+
+Layout/SpaceAroundEqualsInParameterDefault:
+  Enabled: true
+
+Layout/SpaceAroundKeyword:
+  Enabled: true
+
+Layout/SpaceAroundOperators:
+  Enabled: true
+
+Layout/SpaceBeforeFirstArg:
+    Enabled: true
+
+# Defining a method with parameters needs parentheses.
+Style/MethodDefParentheses:
+  Enabled: true
+
+# str = "abcda"
+# str.gsub!("c", "b") # will raise a error, we cannot change string literal
+Style/FrozenStringLiteralComment:
+  Enabled: true
+  EnforcedStyle: always
+  Include:
+    - 'app/**/*'
+    - 'lib/**/*'
+
+# Use `foo {}` not `foo{}`.
+Layout/SpaceBeforeBlockBraces:
+  Enabled: true
+
+# Use `foo { bar }` not `foo {bar}`.
+Layout/SpaceInsideBlockBraces:
+  Enabled: true
+
+# Use `{ a: 1 }` not `{a:1}`.
+Layout/SpaceInsideHashLiteralBraces:
+  Enabled: true
+
+Layout/SpaceInsideParens:
+  Enabled: true
+
+# Check quotes usage according to lint rule below.
+# Style/StringLiterals:
+#   Enabled: true
+#   EnforcedStyle: double_quotes
+
+# Detect hard tabs, no hard tabs.
+Layout/Tab:
+  Enabled: true
+
+# Blank lines should not have any spaces.
+Layout/TrailingBlankLines:
+  Enabled: true
+
+# No trailing whitespace.
+Layout/TrailingWhitespace:
+  Enabled: true
+
+# Use quotes for string literals when they are enough.
+# Style/UnneededPercentQ:
+#   Enabled: true
+
+# Align `end` with the matching keyword or starting expression except for
+# assignments, where it should be aligned with the LHS.
+Lint/EndAlignment:
+  Enabled: true
+  EnforcedStyleAlignWith: variable
+
+# Use my_method(my_arg) not my_method( my_arg ) or my_method my_arg.
+Lint/RequireParentheses:
+  Enabled: true
+

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.1
+before_install: gem install bundler -v 1.16.0.pre.2

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in csvsql.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,57 @@
+PATH
+  remote: .
+  specs:
+    csvsql (0.1.0)
+      sqlite3 (~> 1.3.13)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.0)
+    coderay (1.1.2)
+    diff-lcs (1.3)
+    method_source (0.9.0)
+    parallel (1.12.1)
+    parser (2.5.1.2)
+      ast (~> 2.4.0)
+    powerpack (0.1.2)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    rainbow (3.0.0)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.1)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.1)
+    rubocop (0.52.1)
+      parallel (~> 1.10)
+      parser (>= 2.4.0.2, < 3.0)
+      powerpack (~> 0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-progressbar (1.9.0)
+    sqlite3 (1.3.13)
+    unicode-display_width (1.4.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.16.a)
+  csvsql!
+  pry
+  rspec (~> 3.0)
+  rubocop (~> 0.52.1)
+
+BUNDLED WITH
+   1.16.0.pre.2

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 jiangzhi.xie
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,112 @@
+Csvsql
+======
+
+Use SQL to query your CSV file, and return a new CSV.
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'csvsql'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install csvsql
+
+## Usage
+
+### Example CSV `mydata.csv`
+
+```csv
+name,total:int,created_at:datetime
+a,12,2018-1-1 11:22
+b,21,2018-3-1 10:20
+c,39,2018-1-19 20:10
+```
+
+The first line is title for each columns. We use `:` split the title name and column type. the column type is `varchar(255)` if you did give a type.
+
+There are same types: `int`, `integer` `float`, `double`, `date`, `datetime`, `varchar(255)`.
+
+All sqlite3 type is supported, but maybe some type cannot convert to a sqlite3 value type.
+
+### Query from file & Output
+
+```
+# csvsql [options] SQL
+csvsql -h # Get help
+```
+
+```
+$ csvsql -i mydata.csv "select * from csv where created_at > '2018-1-1' and total > 20" > /tmp/result.csv
+# name:varchar(255),total:int,created_at:datetime
+# b,21,2018-3-1 10:20:00
+# c,39,2018-1-19 20:10:00
+
+csvsql -i mydata.csv "select count(*) as total_record from csv where created_at > '2018-1-1'" > /tmp/result.csv
+# total_record:integer
+# 2
+
+csvsql -i mydata.csv "select name, total from csv where total < 30" > /tmp/result.csv
+# name:varchar(255),total:integer
+# a,12
+# b,21
+```
+
+### Query from stdin
+
+If not give a csv by `-i`, we will get the input from stdin.
+
+```
+csvsql -i mydata.csv "select name, total from csv where total < 30" | csvsql "select count(*) as count from csv"
+# count:integer
+# 2
+```
+
+### Cache CSV data
+
+It will save the CSV data to a tempfile. we use `~/.csvsql_cache` folder to save the cache
+
+```
+csvsql -i large.csv -t "select count(*) from csv"
+
+# the second, it will be fast.
+csvsql -i large.csv -t "select count(*) from csv"
+```
+
+### Clear Cache
+
+This command will remove all data in the `~/.csvsql_cache`
+
+```
+csvsql --clear
+```
+
+
+## Performance TODO
+
+* 10,000 lines
+* 10,000 lines with cache
+* 1,000,000 lines
+* 1,000,000 lines with cache
+
+
+## Development
+
+* Make sure your code has some testing.
+* Run `rubocop -a` before commit your code.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/csvsql.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "csvsql"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/csvsql.gemspec

@@ -0,0 +1,30 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "csvsql/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "csvsql"
+  spec.version       = Csvsql::VERSION
+  spec.authors       = ["jiangzhi.xie"]
+  spec.email         = ["xiejiangzhi@gmail.com"]
+
+  spec.summary       = %q{Process csv with SQL.}
+  spec.description   = %q{Process csv with SQL.}
+  spec.homepage      = "https://github.com/xiejiangzhi/csvsql"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "sqlite3", "~> 1.3.13"
+
+  spec.add_development_dependency "bundler", "~> 1.16.a"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "rubocop", '~> 0.52.1'
+end

data/exe/csvsql

@@ -0,0 +1,34 @@
+#! /usr/bin/env ruby
+
+require 'optparse'
+
+lib = File.expand_path("../../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'csvsql'
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: csvsql [options] SQL"
+
+  opts.on('-i', '--input path', "CSV file path, optional. read from stdin if no give") do |path|
+    options[:csv_path] = path
+  end
+
+  opts.on('-c', '--use-cache', "Cache data in ~/.csvsql_cache. it will still reload if file was changed") do
+    options[:use_cache] = true
+  end
+
+  opts.on('--clear-cache', "Clear all cache data") do
+    options[:clear_cache] = true
+  end
+end.parse!
+
+if options[:clear_cache]
+  Csvsql::Db.clear_cache!
+  puts "Completed clear cache."
+  exit
+end
+
+csv_data = options[:csv_path] || StringIO.new($stdin.read)
+
+puts Csvsql.execute(ARGV[0], csv_data, use_cache: options[:use_cache])

data/lib/csvsql/db.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module Csvsql
+  class Db
+    BATCH_LINES = 10000
+    CACHE_DIR = File.join(Dir.home, '.csvsql_cache')
+
+    FileUtils.mkdir_p(CACHE_DIR) unless Dir.exists?(CACHE_DIR)
+
+    attr_reader :use_cache, :csv_path, :csv_io, :db
+
+    def self.clear_cache!
+      require 'fileutils'
+      FileUtils.rm_f(Dir.glob(File.join(CACHE_DIR, '*')))
+    end
+
+    def initialize(use_cache: false)
+      @db = nil
+      @csv_path = nil
+      @use_cache = use_cache
+    end
+
+    # action:
+    #   raise: default
+    #   exit
+    def sql_error_action=(action)
+      @sql_error_action = action.to_sym
+    end
+
+    def execute(sql)
+      db.execute(sql)
+    rescue SQLite3::SQLException => e
+      process_sql_error(sql, e)
+    end
+
+    def prepare(sql)
+      db.prepare(sql)
+    rescue SQLite3::SQLException => e
+      process_sql_error(sql, e)
+    end
+
+    def import(csv_data_or_path)
+      case csv_data_or_path
+      when StringIO, IO
+        @csv_io = csv_data_or_path
+      else
+        @csv_path = csv_data_or_path
+      end
+      @db = SQLite3::Database.new(get_db_path(@csv_path))
+
+      tables = db.execute("SELECT name FROM sqlite_master WHERE type='table';").first
+      unless tables && tables.include?('csv')
+        init_db_by_csv(@csv_io ? CSV.new(@csv_io) : CSV.open(@csv_path))
+      end
+      true
+    end
+
+    private
+
+    def parser_header(csv_header)
+      csv_header.map do |col, r|
+        name, type = col.strip.split(':')
+        [name, (type || 'varchar(255)').downcase.to_sym]
+      end
+    end
+
+    def init_db_by_csv(csv)
+      header = parser_header(csv.readline)
+
+      cols = header.map { |name, type| "#{name} #{type}" }.join(', ')
+      sql = "CREATE TABLE csv (#{cols});"
+      execute sql
+
+      cache = []
+      col_names = header.map(&:first)
+      csv.each do |line|
+        if cache.length > BATCH_LINES then
+          import_lines(cache, col_names)
+          cache.clear
+        else
+          cache << line.each_with_index.map { |v, i| format_sql_val(v, header[i][1]) }
+        end
+      end
+      import_lines(cache, col_names) unless cache.empty?
+      db
+    end
+
+    def import_lines(lines, col_names)
+      sql = "INSERT INTO csv (#{col_names.join(', ')}) VALUES "
+      values = lines.map { |line| "(#{line.join(',')})" }.join(', ')
+      execute sql + values
+    end
+
+    def format_sql_val(val, type)
+      case type
+      when :int, :integer then val.to_i
+      when :float, :double then val.to_f
+      when :date then "'#{Date.parse(val).to_s}'"
+      when :datetime then "'#{Time.parse(val).strftime('%F %T')}'"
+      else
+        "'#{val.gsub("'", "''")}'"
+      end
+    rescue => e
+      process_sql_error("Parse val: #{val}", e)
+    end
+
+    def process_sql_error(sql, err)
+      $stderr.puts(sql)
+
+      if @error_action == :exit
+        $stderr.puts(e.message)
+        exit
+      else
+        raise err
+      end
+    end
+
+    def get_db_path(csv_path)
+      csv_path = csv_path || ''
+      return '' unless File.exist?(csv_path)
+
+      if use_cache
+        stat = File.stat(csv_path)
+        filename = Digest::SHA2.hexdigest(File.absolute_path(csv_path)) + '.cache'
+        file_stat = [File.absolute_path(csv_path), stat.size, stat.ctime].join("\n")
+        stat_path = File.join(CACHE_DIR, filename.gsub(/\.cache$/, '.stat'))
+        cache_path = File.join(CACHE_DIR, filename)
+
+        if File.exist?(stat_path)
+          if File.read(stat_path) == file_stat
+            cache_path
+          else
+            FileUtils.rm(cache_path)
+            cache_path
+          end
+        else
+          File.write(stat_path, file_stat)
+          cache_path
+        end
+      else
+        ''
+      end
+    end
+  end
+end

data/lib/csvsql/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Csvsql
+  VERSION = "0.1.0"
+end

data/lib/csvsql.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "csvsql/version"
+
+require 'csv'
+require 'sqlite3'
+
+require 'csvsql/db'
+
+module Csvsql
+  def self.execute(sql, csv_data, opts = {})
+    csvdb = Csvsql::Db.new(opts)
+    csvdb.import(csv_data)
+    pst = csvdb.prepare(sql)
+    CSV.generate do |csv|
+      csv << pst.columns.zip(pst.types).map { |c| c.compact.join(':') }
+      pst.each { |line| csv << line }
+    end
+  end
+end

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification
+name: csvsql
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- jiangzhi.xie
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-07-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.13
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.13
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.16.a
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.16.a
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.52.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.52.1
+description: Process csv with SQL.
+email:
+- xiejiangzhi@gmail.com
+executables:
+- csvsql
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- bin/console
+- bin/setup
+- csvsql.gemspec
+- exe/csvsql
+- lib/csvsql.rb
+- lib/csvsql/db.rb
+- lib/csvsql/version.rb
+homepage: https://github.com/xiejiangzhi/csvsql
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.13
+signing_key: 
+specification_version: 4
+summary: Process csv with SQL.
+test_files: []