csvsql 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 77f9bf18a9ec8773b4015b456cd4dc789bebd36d
-  data.tar.gz: 6c18bed0e815307ffa18cd8b9ef80a774abecaa9
+  metadata.gz: 70fa4c1c358f92a85d5f1cd0502081297d1d27cc
+  data.tar.gz: 0bf5efcd968a8965c8308b75e39ae82e43c8d29e
 SHA512:
-  metadata.gz: 567347889397e591db8a04dc8dd9b8154e4915eb85b88575f62767a7e78bc578e2282d8250cd27f5947cc7462e3bd37f6869c455ec76ada203be4c8e03b1da7b
-  data.tar.gz: 4c345a17fb5b380d6039fbb1bfad0609596a1703c8dca07b65cd917f5bfa0b77d49e7d3b41181d6f6cf7195b8ebc8f5d869099aebfc0d2b8dc190de64e303656
+  metadata.gz: 88ddda2f38d83fd4defc23bbfd0d596517f600aacf30963a63b1a15c7f5dca73d7dfe977b0d8b80bff597238ee8e515f559480afb6d54c6a3c2b30e7d6626f24
+  data.tar.gz: d51eb995c1acfe9420153e3acb25f91c06137276f88dd3d67d60d7807395acefd9a7d9403030dfde5a22ab3c0e3492e5de44d479478eca26288f0b475e30b4e4

data/.gitignore

@@ -9,5 +9,7 @@
 
 # rspec failure tracking
 .rspec_status
+/spec/examples.txt
 
 *.swp
+*.gem

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    csvsql (0.1.0)
+    csvsql (0.1.1)
       sqlite3 (~> 1.3.13)
 
 GEM

data/README.md

@@ -75,10 +75,10 @@ csvsql -i mydata.csv "select name, total from csv where total < 30" | csvsql "se
 It will save the CSV data to a tempfile. we use `~/.csvsql_cache` folder to save the cache
 
 ```
-csvsql -i large.csv -t "select count(*) from csv"
+csvsql -i large.csv -c "select count(*) from csv"
 
 # the second, it will be fast.
-csvsql -i large.csv -t "select count(*) from csv"
+csvsql -i large.csv -c "select count(*) from csv"
 ```
 
 ### Clear Cache
@@ -86,7 +86,7 @@ csvsql -i large.csv -t "select count(*) from csv"
 This command will remove all data in the `~/.csvsql_cache`
 
 ```
-csvsql --clear
+csvsql --clear-cache
 ```
 
 

data/exe/csvsql

@@ -8,7 +8,8 @@ require 'csvsql'
 
 options = {}
 OptionParser.new do |opts|
-  opts.banner = "Usage: csvsql [options] SQL"
+  opts.banner = "Csvsql #{Csvsql::VERSION}\nUsage: csvsql [options] SQL"
+  opts.version = Csvsql::VERSION
 
   opts.on('-i', '--input path', "CSV file path, optional. read from stdin if no give") do |path|
     options[:csv_path] = path
@@ -18,9 +19,20 @@ OptionParser.new do |opts|
     options[:use_cache] = true
   end
 
+  opts.on(
+    '-b', '--batch-rows n',
+    "How many rows to import per batch. Default value is #{Csvsql::Db::BATCH_ROWS}"
+  ) do |n|
+    options[:batch_rows] = n.to_i
+  end
+
   opts.on('--clear-cache', "Clear all cache data") do
     options[:clear_cache] = true
   end
+
+  opts.on('--debug', "Print debug info") do
+    options[:debug] = true
+  end
 end.parse!
 
 if options[:clear_cache]
@@ -29,6 +41,10 @@ if options[:clear_cache]
   exit
 end
 
+if options[:debug]
+  Csvsql::Tracker.tracker = Csvsql::Tracker.new(Logger.new($stdout))
+end
+
 csv_data = options[:csv_path] || StringIO.new($stdin.read)
 
-puts Csvsql.execute(ARGV[0], csv_data, use_cache: options[:use_cache])
+puts Csvsql.execute(ARGV[0], csv_data, use_cache: options[:use_cache], batch_rows: options[:batch_rows])

data/lib/csvsql/db.rb

@@ -2,146 +2,149 @@
 
 require 'digest'
 
-module Csvsql
-  class Db
-    BATCH_LINES = 10000
-    CACHE_DIR = File.join(Dir.home, '.csvsql_cache')
+class Csvsql::Db
+  BATCH_ROWS = 10000
+  CACHE_DIR = File.join(Dir.home, '.csvsql_cache')
+  FileUtils.mkdir_p(CACHE_DIR) unless Dir.exists?(CACHE_DIR)
 
-    FileUtils.mkdir_p(CACHE_DIR) unless Dir.exists?(CACHE_DIR)
+  attr_reader :use_cache, :csv_path, :csv_io, :db, :batch_rows
 
-    attr_reader :use_cache, :csv_path, :csv_io, :db
+  def self.clear_cache!
+    require 'fileutils'
+    FileUtils.rm_f(Dir.glob(File.join(CACHE_DIR, '*')))
+  end
 
-    def self.clear_cache!
-      require 'fileutils'
-      FileUtils.rm_f(Dir.glob(File.join(CACHE_DIR, '*')))
-    end
+  def initialize(use_cache: false, batch_rows: nil)
+    @db = nil
+    @csv_io = nil
+    @csv_path = nil
+    @use_cache = use_cache
+    @batch_rows = batch_rows || BATCH_ROWS
+  end
 
-    def initialize(use_cache: false)
-      @db = nil
-      @csv_path = nil
-      @use_cache = use_cache
-    end
+  # action:
+  #   raise: default
+  #   exit
+  def sql_error_action=(action)
+    @sql_error_action = action.to_sym
+  end
 
-    # action:
-    #   raise: default
-    #   exit
-    def sql_error_action=(action)
-      @sql_error_action = action.to_sym
-    end
+  def execute(sql)
+    db.execute(sql)
+  rescue SQLite3::SQLException => e
+    process_sql_error(sql, e)
+  end
 
-    def execute(sql)
-      db.execute(sql)
-    rescue SQLite3::SQLException => e
-      process_sql_error(sql, e)
-    end
+  def prepare(sql)
+    db.prepare(sql)
+  rescue SQLite3::SQLException => e
+    process_sql_error(sql, e)
+  end
 
-    def prepare(sql)
-      db.prepare(sql)
-    rescue SQLite3::SQLException => e
-      process_sql_error(sql, e)
+  def import(csv_data_or_path)
+    case csv_data_or_path
+    when StringIO, IO
+      @csv_io = csv_data_or_path
+    else
+      @csv_path = csv_data_or_path
     end
+    @db = SQLite3::Database.new(get_db_path(@csv_path))
 
-    def import(csv_data_or_path)
-      case csv_data_or_path
-      when StringIO, IO
-        @csv_io = csv_data_or_path
-      else
-        @csv_path = csv_data_or_path
-      end
-      @db = SQLite3::Database.new(get_db_path(@csv_path))
-
-      tables = db.execute("SELECT name FROM sqlite_master WHERE type='table';").first
-      unless tables && tables.include?('csv')
-        init_db_by_csv(@csv_io ? CSV.new(@csv_io) : CSV.open(@csv_path))
-      end
-      true
+    tables = db.execute("SELECT name FROM sqlite_master WHERE type='table';").flatten
+    unless tables.include?('csv')
+      init_db_by_csv(@csv_io ? CSV.new(@csv_io) : CSV.open(@csv_path))
     end
+    true
+  end
 
-    private
+  private
 
-    def parser_header(csv_header)
-      csv_header.map do |col, r|
-        name, type = col.strip.split(':')
-        [name, (type || 'varchar(255)').downcase.to_sym]
-      end
+  def parser_header(csv_header)
+    csv_header.map do |col, r|
+      name, type = col.strip.split(':')
+      [name, (type || 'varchar(255)').downcase.to_sym]
     end
+  end
 
-    def init_db_by_csv(csv)
-      header = parser_header(csv.readline)
+  def init_db_by_csv(csv)
+    header = parser_header(csv.readline)
 
-      cols = header.map { |name, type| "#{name} #{type}" }.join(', ')
-      sql = "CREATE TABLE csv (#{cols});"
-      execute sql
+    cols = header.map { |name, type| "#{name} #{type}" }.join(', ')
+    sql = "CREATE TABLE csv (#{cols});"
+    execute sql
 
-      cache = []
-      col_names = header.map(&:first)
-      csv.each do |line|
-        if cache.length > BATCH_LINES then
-          import_lines(cache, col_names)
-          cache.clear
-        else
-          cache << line.each_with_index.map { |v, i| format_sql_val(v, header[i][1]) }
-        end
+    cache = []
+    col_names = header.map(&:first)
+    Csvsql::Tracker.commit(:import_csv)
+    csv.each do |line|
+      cache << line.each_with_index.map { |v, i| format_sql_val(v, header[i][1]) }
+
+      if cache.length >= batch_rows then
+        import_lines(cache, col_names)
+        cache.clear
       end
-      import_lines(cache, col_names) unless cache.empty?
-      db
     end
+    import_lines(cache, col_names) unless cache.empty?
+    Csvsql::Tracker.commit(:import_csv)
+    db
+  end
 
-    def import_lines(lines, col_names)
-      sql = "INSERT INTO csv (#{col_names.join(', ')}) VALUES "
-      values = lines.map { |line| "(#{line.join(',')})" }.join(', ')
-      execute sql + values
+  def import_lines(lines, col_names)
+    sql = Csvsql::Tracker.commit(:generate_import_sql) do
+      s = "INSERT INTO csv (#{col_names.join(', ')}) VALUES "
+      s += lines.map { |line| "(#{line.join(',')})" }.join(', ')
     end
+    Csvsql::Tracker.commit(:execute_import_sql) { execute sql }
+  end
 
-    def format_sql_val(val, type)
-      case type
-      when :int, :integer then val.to_i
-      when :float, :double then val.to_f
-      when :date then "'#{Date.parse(val).to_s}'"
-      when :datetime then "'#{Time.parse(val).strftime('%F %T')}'"
-      else
-        "'#{val.gsub("'", "''")}'"
-      end
-    rescue => e
-      process_sql_error("Parse val: #{val}", e)
+  def format_sql_val(val, type)
+    case type
+    when :int, :integer then val.to_i
+    when :float, :double then val.to_f
+    when :date then "'#{Date.parse(val).to_s}'"
+    when :datetime then "'#{Time.parse(val).strftime('%F %T')}'"
+    else
+      "'#{val.gsub("'", "''")}'"
     end
+  rescue => e
+    process_sql_error("Parse val: #{val}", e)
+  end
 
-    def process_sql_error(sql, err)
-      $stderr.puts(sql)
+  def process_sql_error(sql, err)
+    $stderr.puts(sql)
 
-      if @error_action == :exit
-        $stderr.puts(e.message)
-        exit
-      else
-        raise err
-      end
+    if @error_action == :exit
+      $stderr.puts(e.message)
+      exit
+    else
+      raise err
     end
+  end
 
-    def get_db_path(csv_path)
-      csv_path = csv_path || ''
-      return '' unless File.exist?(csv_path)
-
-      if use_cache
-        stat = File.stat(csv_path)
-        filename = Digest::SHA2.hexdigest(File.absolute_path(csv_path)) + '.cache'
-        file_stat = [File.absolute_path(csv_path), stat.size, stat.ctime].join("\n")
-        stat_path = File.join(CACHE_DIR, filename.gsub(/\.cache$/, '.stat'))
-        cache_path = File.join(CACHE_DIR, filename)
-
-        if File.exist?(stat_path)
-          if File.read(stat_path) == file_stat
-            cache_path
-          else
-            FileUtils.rm(cache_path)
-            cache_path
-          end
+  def get_db_path(csv_path)
+    csv_path = csv_path || ''
+    return '' unless File.exist?(csv_path)
+
+    if use_cache
+      stat = File.stat(csv_path)
+      filename = Digest::SHA2.hexdigest(File.absolute_path(csv_path)) + '.cache'
+      file_stat = [File.absolute_path(csv_path), stat.size, stat.ctime].join("\n")
+      stat_path = File.join(CACHE_DIR, filename.gsub(/\.cache$/, '.stat'))
+      cache_path = File.join(CACHE_DIR, filename)
+
+      if File.exist?(stat_path)
+        if File.read(stat_path) == file_stat
+          cache_path
         else
-          File.write(stat_path, file_stat)
+          FileUtils.rm(cache_path)
           cache_path
         end
       else
-        ''
+        File.write(stat_path, file_stat)
+        cache_path
       end
+    else
+      ''
     end
   end
 end

data/lib/csvsql/tracker.rb

@@ -0,0 +1,44 @@
+require 'logger'
+
+class Csvsql::Tracker
+  attr_reader :stats, :logger
+
+  def self.tracker
+    @tracker ||= new
+  end
+
+  def self.tracker=(t)
+    @tracker = t
+  end
+
+  def self.commit(*args, &block)
+    tracker.commit(*args, &block)
+  end
+
+  def initialize(logger = Logger.new('/dev/null'))
+    @stats = {}
+    @logger = logger
+  end
+
+  def commit(id, output: true, &block)
+    id = id.to_s
+    old = stats[id]
+    stats[id] = get_stat
+
+    if block
+      block.call.tap { commit(id) }
+    elsif output && old
+      logger.info("[#{id}] #{compare_stat(old, stats[id])}")
+    end
+  end
+
+  private
+
+  def get_stat
+    { time: Time.now }
+  end
+
+  def compare_stat(old, new)
+    "Time cost: #{((new[:time] - old[:time]) * 1000000).to_i / 1000}ms"
+  end
+end

data/lib/csvsql/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Csvsql
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/csvsql.rb

@@ -6,15 +6,20 @@ require 'csv'
 require 'sqlite3'
 
 require 'csvsql/db'
+require 'csvsql/tracker'
 
 module Csvsql
   def self.execute(sql, csv_data, opts = {})
     csvdb = Csvsql::Db.new(opts)
     csvdb.import(csv_data)
-    pst = csvdb.prepare(sql)
+    pst = Csvsql::Tracker.commit(:execute_query_sql) do
+      csvdb.prepare(sql)
+    end
+    Csvsql::Tracker.commit(:output_format)
     CSV.generate do |csv|
       csv << pst.columns.zip(pst.types).map { |c| c.compact.join(':') }
       pst.each { |line| csv << line }
-    end
+    end.tap { Csvsql::Tracker.commit(:output_format) }
   end
 end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csvsql
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - jiangzhi.xie
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-07-18 00:00:00.000000000 Z
+date: 2018-07-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sqlite3
@@ -102,6 +102,7 @@ files:
 - exe/csvsql
 - lib/csvsql.rb
 - lib/csvsql/db.rb
+- lib/csvsql/tracker.rb
 - lib/csvsql/version.rb
 homepage: https://github.com/xiejiangzhi/csvsql
 licenses: