muzzy 0.1.11 → 0.1.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9fafdd625f30a8271efa806cf9458bb8c1b61ef4
-  data.tar.gz: efc95b3c3c81c9d84b679b0adc4484588a191b62
+  metadata.gz: 0cb78304c4f64a469d72f3fc41d6482dbf0c3d78
+  data.tar.gz: 5241fc24a5787b37fa3057317a8535c95901ad80
 SHA512:
-  metadata.gz: cafee7cee7b9f4d1082d5459395d2f33b713f418108fc50b6409306cbc832f5f9abea1891185c79da4a61b3ba3411f72e2cbc80a5ccc9c1bfdfa333d3188b49b
-  data.tar.gz: 661f7617630ff85e2c8effa0ff0e10f67560213d6a0187a75723038fa3e2c0e139f3af40cbfe15da4373fca710d4a7fb567ff0549d00f1b89519e81b5e792c2a
+  metadata.gz: 541b75bbfacee9521fdf96e9029d1a765f520771482862b662b1b77e37b09dd634f2371f7fae4597a725f0d4ed98b517730000b9ca152a9be129f59920d516c7
+  data.tar.gz: b0efa731b1bbb93cfb5406bd2b2edb799831a9d5ea4f1f0e1fea8c5a95cc61a887259b1fdd3d79c8fa2e44fdb23290e89d5f899fec88b808ab17209db28c2e2b

data/exe/muzzy

@@ -12,9 +12,19 @@ Version = Muzzy::VERSION
 ## check environments
 kakasi_path = File.absolute_path(File.expand_path('..', __FILE__) + "/../vendor/bin/kakasi")
 unless File.exists?(kakasi_path)
-  puts "cannot find kakasi. please check kakasi installation is successed."
-  puts "expect path: #{kakasi_path}"
-  puts "If you have not execute `muzzy_setup` yet, please run `muzzy_setup` first."
+  $stderr.puts "cannot find kakasi. please check kakasi installation is successed."
+  $stderr.puts "expect path: #{kakasi_path}"
+  $stderr.puts "If you have not execute `muzzy_setup` yet, please run `muzzy_setup` first."
+  exit 1
+end
+mysql_cmd = `which mysql`.to_s.chomp
+if mysql_cmd == ''
+  $stderr.puts "cannot find mysql client"
+  exit 1
+end
+mysqlimport_cmd = `which mysqlimport`.to_s.chomp
+if mysqlimport_cmd == ''
+  $stderr.puts "cannot find mysqlimport command"
   exit 1
 end
 
@@ -33,25 +43,13 @@ options = {
   },
 }
 
-mysql_cmd = `which mysql`.to_s.chomp
-if mysql_cmd == ''
-  puts "cannot find mysql client"
-  exit 1
-end
-
-mysqlimport_cmd = `which mysqlimport`.to_s.chomp
-if mysqlimport_cmd == ''
-  puts "cannot find mysqlimport command"
-  exit 1
-end
-
 def check_file(file)
   if File.directory?(file)
-    puts "muzzy: #{file}: is directory"
+    $stderr.puts "muzzy: #{file}: is directory"
     exit Errno::ENOENT::Errno
   end
   unless File.exists?(file)
-    puts "muzzy: #{file}: No such file or directory"
+    $stderr.puts "muzzy: #{file}: No such file or directory"
     exit Errno::ENOENT::Errno
   end
 end
@@ -75,24 +73,23 @@ ARGV.options do |opt|
     }
 
     if ARGV[0].to_s.length > 0 && ARGV[0].to_s[0] != '-'
-      check_file(ARGV[0])
       options[:file] = ARGV[0]
+    elsif ARGV.last.to_s.length > 0 && ARGV.last.to_s[0] != '-'
+      options[:file] = ARGV.last
     else
-      opt.on('-f', '--file [FILEPATH]', 'path to target file') {|v|
-        check_file(v)
-        options[:file] = v
-      }
+      opt.on('-f', '--file [FILEPATH]', 'path to target file') {|v| options[:file] = v }
     end
+    check_file(options[:file])
 
     opt.on('-v', '--version') {
-      puts opt.ver
+      $stdout.puts opt.ver
       exit
     }
 
     opt.parse!
 
     if options[:file].nil? || options[:file] == ''
-      puts opt.help
+      $stdout.puts opt.help
       exit 1
     end
   rescue => e
@@ -102,118 +99,44 @@ ARGV.options do |opt|
 end
 
 if options[:mysql_config][:database] !~ /\A(\w)+\z/
-  puts "illegal database"
+  $stderr.puts "illegal database name"
   exit 1
 end
 
-## confirm database, if not, create it.
-mysql_cmd_list = [mysql_cmd, '-u', options[:mysql_config][:user], '-h', options[:mysql_config][:host]]
-if options[:mysql_config][:use_password]
-  mysql_cmd_list.push('-p')
-end
-
-confirm_database_cmd_list = mysql_cmd_list + ['-e', '"' + "SHOW DATABASES LIKE '#{options[:mysql_config][:database]}'" + '"']
-if options[:verbose]
-  puts confirm_database_cmd_list.join(' ')
-end
-
-confirm_database_res = `#{confirm_database_cmd_list.join(' ')}`.chomp
-if confirm_database_res == ''
-  puts "creating database #{options[:mysql_config][:database]}"
-  system(*mysql_cmd_list, '-e', "CREATE DATABASE #{options[:mysql_config][:database]}")
-  if $? == 0
-    puts "creating database #{options[:mysql_config][:database]} done"
-  end
-end
-
-## get first and second rows
-header_row = []
-first_row = [] # first row means first data row
-fields_terminated_by = ","
-
-
-if options[:file] =~ /\.tsv\z/
-  # tsv
-  header_row, first_row = Muzzy::Util.fetch_header_and_first_row(options[:file], "\t")
-  fields_terminated_by = "\t"
-else
-  # csv(,) or csv(\t) or something
-
-  # trying to csv
-  csv_header_row, csv_first_row = Muzzy::Util.fetch_header_and_first_row(options[:file], ",")
-  # trying to tsv
-  tsv_header_row, tsv_first_row = Muzzy::Util.fetch_header_and_first_row(options[:file], "\t")
-
-  if csv_header_row == -1 && tsv_header_row == -1
-    puts "illegal csv file."
-    exit 1
-  end
-
-  if csv_header_row == -1 || tsv_header_row == -1
-    if csv_header_row == -1
-      # tsv
-      header_row, first_row = tsv_header_row, tsv_first_row
-      fields_terminated_by = "\t"
-    else
-      # csv
-      header_row, first_row = csv_header_row, csv_first_row
-    end
-  else
-    if csv_header_row.length > tsv_header_row.length
-      # csv
-      header_row, first_row = csv_header_row, csv_first_row
-    else
-      # tsv
-      header_row, first_row = tsv_header_row, tsv_first_row
-      fields_terminated_by = "\t"
-    end
-    if csv_header_row.length == tsv_header_row.length
-      if csv_header_row.length == 1 && tsv_first_row.length == 1
-        # single col file
-        header_row, first_row = csv_header_row, csv_first_row
-      else
-        puts "illegal file"
-        exit 1
-      end
-    end
-  end
+filetype_detector = Muzzy::FiletypeDetector.new(options[:file])
+if filetype_detector.unknown?
+  $stderr.puts "illegal file"
+  exit 1
 end
-header_row = header_row.map(&:to_s)
 
+first_row = filetype_detector.first_row
+second_row = filetype_detector.second_row
+fields_terminated_by = filetype_detector.tsv? ? "\t" : ','
 
-# guess header_row is data row or not
-header_row_is_header = nil
-
-if header_row.any?{|str| str.match(/_id/) }
-  header_row_is_header = true
-elsif header_row.any?{|str| str.match(/NULL/) }
-  header_row_is_header = false
-elsif header_row.any?{|str| str.nil? || str == '' }
-  header_row_is_header = false
-elsif header_row.map(&:to_i).select{|x| x <= 0}.count != first_row.map(&:to_i).select{|x| x <= 0}.count
-  # i guess number col count is different, header_row is header.
-  header_row_is_header = true
-else
+# detect first_row is header or data row
+first_row_is_header = Muzzy::HeaderDetector.detect([first_row, second_row])
+if first_row_is_header == nil
   # cannot judge header is header or not
-  puts "header row is"
-  puts header_row.join(',')
-  puts "header row is HEADER? [y/n]"
+  puts "first row is"
+  puts first_row.join(',')
+  puts "first row is HEADER? [y/n]"
   loop do
     y_or_n = $stdin.gets.to_s.chomp
     if y_or_n.match(/\A[Yy]\z/)
-      header_row_is_header = true
+      first_row_is_header = true
       break
     end
     if y_or_n.match(/\A[Nn]\z/)
-      header_row_is_header = false
+      first_row_is_header = false
       break
     end
     puts "plase enter y or n"
   end
 end
 
-if header_row_is_header
-  header_row = header_row.map do |str|
+
+if first_row_is_header
+  first_row = first_row.map do |str|
     std_out = Open3.capture2('echo', str)[0]
     Open3.capture2(kakasi_path, '-Ja', '-Ha', '-Ka', '-Ea', '-i', 'utf8', '-o', 'utf8', stdin_data: std_out)[0]
   end.map(&:chomp).map do |x|
@@ -222,19 +145,35 @@ if header_row_is_header
   end
 end
 
+# TODO ヘッダが空白含んでたりするやつとかをなんとかする
+col_data_types = []
 Cell = Struct.new(:type, :name)
-if header_row_is_header
-  row_data_types = first_row.map.with_index do |str, i|
-    if str.to_s.match(/\A[\d,]+\z/)
-      # number
-      Cell.new('integer', header_row[i])
-    else
-      Cell.new('text', header_row[i])
+if first_row_is_header
+  if first_row
+    col_data_types = first_row.map.with_index do |str, i|
+      if str.to_s.match(/\A[\d,]+\z/)
+        # number
+        Cell.new('integer', first_row[i])
+      else
+        Cell.new('text', first_row[i])
+      end
+    end
+  else
+    col_data_types = first_row.map.with_index do |str, i|
+      colname = first_row[i].gsub(/[,-]/, '')
+      if str.to_s.match(/_id/i) && str.to_s.match(/\A[\w]+\z/i)
+        # number
+        Cell.new('integer', colname)
+      else
+        Cell.new('text', colname)
+      end
     end
   end
 else
-  # header row is data
-  row_data_types = header_row.map.with_index do |str, i|
+  # TODO not create table option
+
+  # first row is data(not header)
+  col_data_types = first_row.map.with_index do |str, i|
     if str.to_s.match(/\A[\d,]+\z/)
       # number
       Cell.new('integer', "col#{i}")
@@ -244,48 +183,45 @@ else
   end
 end
 
-filename = File.basename(options[:file])
-table_name = filename.match(/\A(\w+)(\.\w+)?\z/)[1]
-
-create_table_sql = "CREATE TABLE #{table_name} (#{row_data_types.map{|x| "#{x.name} #{x.type}"}.join(', ')})"
-
-confirm_table_cmd = mysql_cmd_list + [options[:mysql_config][:database], '-e', "\"SHOW CREATE TABLE #{table_name}\"", '2>&1']
-if options[:verbose]
-  puts confirm_table_cmd.join(' ')
-end
+config = {
+  filepath: options[:file],
+  cmd_path: mysql_cmd,
+  mysqlimport_path: mysqlimport_cmd,
+  user: options[:mysql_config][:user],
+  host: options[:mysql_config][:host],
+  # password: options[:mysql_config][:password],
+  use_password: options[:mysql_config][:use_password],
+  database_name: options[:mysql_config][:database],
+}
 
-confirm_table_res = `#{confirm_table_cmd.join(' ')}`
-if confirm_table_res.to_s.match(/\AERROR 1146/)
-  create_database_cmd_list = mysql_cmd_list + [options[:mysql_config][:database], '-e', '"', "#{create_table_sql}", '"']
-  if options[:verbose]
-    puts create_database_cmd_list.join(' ')
+db_adapter = Muzzy::DatabaseAdapter::MysqlAdapter.new(
+  config,
+  verbose: options[:verbose]
+)
+unless db_adapter.confirm_database
+  unless db_adapter.create_database
+    # database create failed
+    exit 1
   end
-  `#{create_database_cmd_list.join(' ')}`
 end
 
-mysql_options = ['-u', options[:mysql_config][:user], '-h', options[:mysql_config][:host]]
-if options[:mysql_config][:use_password]
-  mysql_options.push('-p')
-end
-cmds = [
-  mysqlimport_cmd,
-  *mysql_options,
-  options[:mysql_config][:database],
-  '--local', options[:file],
-  "--ignore-lines=#{header_row_is_header ? 1 : 0}",
-]
-if fields_terminated_by
-  cmds.push("--fields_terminated_by=#{fields_terminated_by}")
-end
-if options[:mysqlimport_config][:delete]
-  cmds.push('--delete')
-end
-cmds.push('--fields_enclosed_by="')
+filename = File.basename(options[:file])
+table_name = filename.match(/\A(\w+)(\.\w+)?\z/)[1]
 
-if options[:verbose]
-  puts cmds.join(' ')
+# confirm table
+unless db_adapter.confirm_table(table_name)
+  # cannot confirm table so create table
+  unless db_adapter.create_table(table_name, col_data_types)
+    # error, cannot create table
+    exit 1
+  end
 end
 
-# mysqlimport data
-system(*cmds)
+# import
+db_adapter.import(table_name, {
+  first_row_is_header: first_row_is_header,
+  fields_terminated_by: fields_terminated_by,
+  delete: options[:mysqlimport_config][:delete]
+})
+
 exit 0

data/lib/muzzy/database_adapters/adapter_base.rb

@@ -0,0 +1,11 @@
+module Muzzy
+  module DatabaseAdapter
+    class AdapterBase
+      def confirm_database; end
+      def create_database; end
+      def confirm_table; end
+      def create_table; end
+      def import; end
+    end
+  end
+end

data/lib/muzzy/database_adapters/mysql_adapter.rb

@@ -0,0 +1,113 @@
+module Muzzy
+  module DatabaseAdapter
+    class MysqlAdapter < AdapterBase
+      def initialize(config, verbose: false)
+        @filepath = config[:filepath]
+        @cmd_path = config[:cmd_path]
+        @import_cmd_path = config[:mysqlimport_path]
+        @user = config[:user]
+        @host = config[:host]
+        @password = config[:password]
+        @use_password = config[:use_password]
+        @database_name = config[:database_name]
+        @verbose = verbose
+      end
+
+      # [Bool] true: exists, false: nothing
+      def confirm_database
+        confirm_database_cmd_list = mysql_cmd_list + ['-e', '"' + "SHOW DATABASES LIKE '#{@database_name}'" + '"']
+        confirm_database_cmd = confirm_database_cmd_list.join(' ')
+        if @verbose
+          $stderr.puts confirm_database_cmd
+        end
+        `#{confirm_database_cmd}`.chomp != ''
+      end
+
+      # [Bool] true: created, false: error happened
+      def create_database
+        $stderr.puts "creating database #{@database_name}"
+
+        create_database_cmd_list = [*mysql_cmd_list, '-e', "'CREATE DATABASE #{@database_name}'"]
+        create_database_cmd = create_database_cmd_list.join(' ')
+
+        if @verbose
+          $stderr.puts create_database_cmd
+        end
+
+        std_out = Open3.capture2(create_database_cmd)[0]
+        if std_out == ''
+          $stderr.puts "creating database #{@database_name} done"
+        else
+          $stderr.puts "error: #{std_out}"
+        end
+
+        std_out == ''
+      end
+
+      # [Bool] true: table exists, false: no table
+      def confirm_table(table_name)
+        confirm_table_cmd_list = [*mysql_cmd_list, @database_name, '-e', "'SHOW CREATE TABLE #{table_name}'"]
+        confirm_table_cmd = confirm_table_cmd_list.join(' ')
+
+        if @verbose
+          $stderr.puts confirm_table_cmd
+        end
+        o, e, status = Open3.capture3(confirm_table_cmd)
+        return status.success?
+      end
+
+      # [Bool] true: table created, false: some error happened
+      def create_table(table_name, col_data_types)
+        create_table_sql = "CREATE TABLE #{table_name} (#{col_data_types.map{|x| "#{x.name} #{x.type}"}.join(', ')})"
+        create_table_cmd_list = [*mysql_cmd_list, @database_name, '-e', '"', "#{create_table_sql}", '"']
+        create_table_cmd = create_table_cmd_list.join(' ')
+        if @verbose
+          $stderr.puts create_table_cmd
+        end
+        std_out, status = Open3.capture2(create_table_cmd)
+        return status.success?
+      end
+
+      # {fields_terminated_by: '', delete: false,}
+      def import(table_name, option)
+        cmds = [*mysqlimport_cmd_list, @database_name, '--local', @filepath]
+        cmds.push "--ignore-lines=#{option[:first_row_is_header] ? 1 : 0}"
+        cmds.push('--fields_enclosed_by="')
+
+        if option[:fields_terminated_by]
+          cmds.push("--fields_terminated_by=#{option[:fields_terminated_by]}")
+        end
+        if option[:delete]
+          cmds.push('--delete')
+        end
+
+        if @verbose
+          $stderr.puts cmds.join(' ')
+        end
+
+        # mysqlimport data
+        system(*cmds)
+      end
+
+      private
+
+      def mysql_cmd_list
+        return @mysql_cmd_list if defined?(@mysql_cmd_list)
+        @mysql_cmd_list = [@cmd_path] + common_param
+      end
+
+      def mysqlimport_cmd_list
+        return @mysqlimport_cmd_list if defined?(@mysqlimport_cmd_list)
+        @mysqlimport_cmd_list = [@import_cmd_path] + common_param
+      end
+
+      def common_param
+        list = ['-u', @user, '-h', @host]
+        if @use_password
+          @list.push('-p')
+        end
+        list
+      end
+    end
+  end
+end

data/lib/muzzy/filetype_detector.rb

@@ -1,4 +1,76 @@
 module Muzzy
+  # read file content and detect filetype is csv or tsv or others
   class FiletypeDetector
+    attr_reader :filepath
+    attr_reader :first_row, :second_row
+    attr_reader :filetype
+
+    def initialize(filepath)
+      @filepath = filepath
+    end
+
+    def tsv?
+      detect || @filetype == 'tsv'
+    end
+
+    def csv?
+      detect || @filetype == 'csv'
+    end
+
+    def unknown?
+      detect || @filetype == 'unknown'
+    end
+
+    private
+
+    def tsv_ext?
+      File.basename(@filepath) =~ /\.tsv\z/
+    end
+
+    def detect
+      return unless @filetype.nil?
+
+      if tsv_ext?
+        @first_row, @second_row = Muzzy::Util.fetch_header_and_first_row(@filepath, "\t")
+        @filetype = 'tsv'
+        return
+      end
+
+      ## csv(,) or csv(\t) or something
+
+      csv_header_row, csv_first_row = Muzzy::Util.fetch_header_and_first_row(@filepath, ",")
+      tsv_header_row, tsv_first_row = Muzzy::Util.fetch_header_and_first_row(@filepath, "\t")
+      if csv_header_row == -1 && tsv_header_row == -1
+        @first_row, @second_row, @filetype = -1, -1, 'unknown'
+        return
+      end
+
+      if csv_header_row == -1
+        @first_row, @second_row, @filetype = tsv_header_row, tsv_first_row, 'tsv'
+        return
+      end
+      if tsv_header_row == -1
+        @first_row, @second_row, @filetype = csv_header_row, csv_first_row, 'csv'
+        return
+      end
+
+      ## rare case
+
+      if csv_header_row.length > tsv_header_row.length
+        @first_row, @second_row, @filetype = csv_header_row, csv_first_row, 'csv'
+        return
+      else
+        @first_row, @second_row, @filetype = tsv_header_row, tsv_first_row, 'tsv'
+        return
+      end
+
+      if csv_header_row.length == 1 && tsv_first_row.length == 1
+        # single col file treat as csv
+        @first_row, @second_row, @filetype = csv_header_row, csv_first_row, 'csv'
+        return
+      end
+
+      @first_row, @second_row, @filetype = -1, -1, 'unknown'
+    end
   end
 end

data/lib/muzzy/header_detector.rb

@@ -0,0 +1,34 @@
+module Muzzy
+  class HeaderDetector
+    # true: first row is header
+    # false: first row is not header
+    # nil: could not detect
+    def self.detect(rows)
+      first_row, second_row = rows || []
+      return nil if first_row.empty?
+
+      if first_row.any?{|str| str.to_s.match(/_id/i) }
+        return true
+      elsif first_row.any?{|str| str.to_s.match(/NULL/) }
+        return false
+      elsif first_row.any?{|str| str.to_s == '' }
+        return false
+      end
+
+      return nil if second_row.empty?
+
+      # I can't detect first_row is header or not, so guess now.
+
+      # general header row is not contain numbers
+      first_row_number_count = first_row.select{|str| str.to_f > 0}.length
+      return false if first_row_number_count > 0
+
+      # If number col count is different, I guess first_row is header.
+      if first_row_number_count != second_row.select{|x| x.to_f > 0}.count
+        return true
+      end
+
+      return nil
+    end
+  end
+end

data/lib/muzzy/version.rb

@@ -1,3 +1,3 @@
 module Muzzy
-  VERSION = "0.1.11"
+  VERSION = "0.1.12"
 end

data/lib/muzzy.rb

@@ -1,5 +1,9 @@
 require 'muzzy/version'
 require_relative 'muzzy/util'
+require_relative 'muzzy/filetype_detector'
+require_relative 'muzzy/header_detector'
+require_relative 'muzzy/database_adapters/adapter_base'
+require_relative 'muzzy/database_adapters/mysql_adapter'
 
 module Muzzy
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: muzzy
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.1.12
 platform: ruby
 authors:
 - vimtaku
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-08-21 00:00:00.000000000 Z
+date: 2018-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -90,7 +90,10 @@ files:
 - exe/muzzy
 - exe/muzzy_setup
 - lib/muzzy.rb
+- lib/muzzy/database_adapters/adapter_base.rb
+- lib/muzzy/database_adapters/mysql_adapter.rb
 - lib/muzzy/filetype_detector.rb
+- lib/muzzy/header_detector.rb
 - lib/muzzy/util.rb
 - lib/muzzy/version.rb
 - muzzy.gemspec