muzzy 0.1.13 → 0.1.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 70311bea7d537c03c4f4a1980a3333701752c0db
-  data.tar.gz: 52a3d53769438167c321c5631cb7baa5e15da5b8
+  metadata.gz: d6d05258a12af946f1c15258b363540856fcdd32
+  data.tar.gz: 41d9d79203216ae03187d1abb1fae0aee0e72d11
 SHA512:
-  metadata.gz: 86d2d36ca0fa3eb1cc41c8a3c604a5f910e9d8006ea941fe94f8a3c013fe41f18dbd2f1b764a7a708bdf04ccb0a54fe81ae6da4f1f8785c86c891ab82aa0739a
-  data.tar.gz: 4ba9b910287dbe31f566a6d396cfad67ff44828d87173e96ca85b95b079b407c04bd2343377bc45f0b9a0e1b9a19130f5d2d45918c871271738d6190a66340f0
+  metadata.gz: 36d777b102c5797d9c74fd7ca9121062949b22b01ca0c597a92ca28335b61de9c09f2843cb2aacb000329f6d2df77c44bc72d16020b6a4758e1efe2aaf0b9314
+  data.tar.gz: 048fdc85d6d4d68c4d3844ab76c7bd8fe5cf500d2bbe996d5e4fb3ad4b48e9d5122aaa7a2146b7f63098c0023fa6b44660a79ae6c9baf45cf9a7da5c7c16fdf2

data/exe/muzzy

@@ -28,7 +28,6 @@ if mysqlimport_cmd == ''
   exit 1
 end
 
-
 options = {
   file: '',
   verbose: false,
@@ -136,55 +135,8 @@ if first_row_is_header == nil
   end
 end
 
-# convert header row to compatible with database table columns
-if first_row_is_header
-  first_row = first_row.map do |str|
-    std_out = Open3.capture2('echo', str)[0]
-    Open3.capture2(kakasi_path, '-Ja', '-Ha', '-Ka', '-Ea', '-i', 'utf8', '-o', 'utf8', stdin_data: std_out)[0]
-  end.map do |x|
-    # kakasi returns ko^do if 'コード' given so replace it to _
-    # space changes to _
-    x.chomp.strip.gsub(/[\^]/, '_').gsub(/\s/, '_')
-  end
-end
-
-# TODO ヘッダが空白含んでたりするやつとかをなんとかする
-col_data_types = []
-Cell = Struct.new(:type, :name)
-if first_row_is_header
-  if first_row
-    col_data_types = first_row.map.with_index do |str, i|
-      if str.to_s.match(/\A[\d,]+\z/)
-        # number
-        Cell.new('integer', first_row[i])
-      else
-        Cell.new('text', first_row[i])
-      end
-    end
-  else
-    col_data_types = first_row.map.with_index do |str, i|
-      colname = first_row[i].gsub(/[,-]/, '')
-      if str.to_s.match(/_id/i) && str.to_s.match(/\A[\w]+\z/i)
-        # number
-        Cell.new('integer', colname)
-      else
-        Cell.new('text', colname)
-      end
-    end
-  end
-else
-  # TODO not create table option
-
-  # first row is data(not header)
-  col_data_types = first_row.map.with_index do |str, i|
-    if str.to_s.match(/\A[\d,]+\z/)
-      # number
-      Cell.new('integer', "col#{i}")
-    else
-      Cell.new('text', "col#{i}")
-    end
-  end
-end
+column_generator = Muzzy::ColumnsGenerator.new(kakasi_path)
+columns = column_generator.generate(first_row_is_header, first_row)
 
 config = {
   filepath: options[:file],
@@ -197,7 +149,7 @@ config = {
   database_name: options[:mysql_config][:database],
 }
 
-db_adapter = Muzzy::DatabaseAdapter::MysqlAdapter.new(
+db_adapter = Muzzy::DatabaseAdapters::MysqlAdapter.new(
   config,
   verbose: options[:verbose]
 )
@@ -214,7 +166,7 @@ table_name = filename.match(/\A(\w+)(\.\w+)?\z/)[1]
 # confirm table
 unless db_adapter.confirm_table(table_name)
   # cannot confirm table so create table
-  unless db_adapter.create_table(table_name, col_data_types)
+  unless db_adapter.create_table(table_name, columns)
     # error, cannot create table
     exit 1
   end

data/lib/muzzy.rb

@@ -2,6 +2,8 @@ require 'muzzy/version'
 require_relative 'muzzy/util'
 require_relative 'muzzy/filetype_detector'
 require_relative 'muzzy/header_detector'
+require_relative 'muzzy/column'
+require_relative 'muzzy/columns_generator'
 require_relative 'muzzy/database_adapters/adapter_base'
 require_relative 'muzzy/database_adapters/mysql_adapter'
 

data/lib/muzzy/column.rb

@@ -0,0 +1,10 @@
+module Muzzy
+  class Column
+    attr_reader :name
+    attr_reader :datatype
+    def initialize(datatype, name)
+      @datatype = datatype
+      @name = name
+    end
+  end
+end

data/lib/muzzy/columns_generator.rb

@@ -0,0 +1,41 @@
+module Muzzy
+  # generate columns from first row data
+  class ColumnsGenerator
+    def initialize(kakasi_path)
+      @kakasi_path = kakasi_path
+    end
+
+    def generate(first_row_is_header, first_row)
+      columns = []
+      # convert header row to compatible with database table columns
+      # ref https://dev.mysql.com/doc/refman/5.6/ja/identifiers.html
+      if first_row_is_header
+        # create column names from japanese headers.
+        # trim invalid chars
+        # noop if data is already ascii
+        coulumn_names = first_row.map do |str|
+          std_out = Open3.capture2('echo', str)[0]
+          Open3.capture2(
+            @kakasi_path,
+            '-Ja', '-Ha', '-Ka', '-Ea', '-i', 'utf8', '-o', 'utf8',
+            stdin_data: std_out
+          )[0]
+        end.map do |x|
+          # kakasi returns ko^do if 'コード' given so replace it to _
+          # space changes to _
+          x.chomp.strip.gsub(/[\^]/, '_').gsub(/\s/, '_')
+        end || first_row.dup
+
+        columns = coulumn_names.map.with_index do |str, i|
+          Muzzy::Column.new('text', coulumn_names[i])
+        end
+      else
+        # first row is data (not header)
+        columns = first_row.map.with_index do |str, i|
+          Muzzy::Column.new('text', "col#{i}")
+        end
+      end
+      columns
+    end
+  end
+end

data/lib/muzzy/database_adapters/adapter_base.rb

@@ -1,10 +1,10 @@
 module Muzzy
-  module DatabaseAdapter
+  module DatabaseAdapters
     class AdapterBase
       def confirm_database; end
       def create_database; end
-      def confirm_table; end
-      def create_table; end
+      def confirm_table(table_name); end
+      def create_table(table_name, columns); end
       def import; end
     end
   end

data/lib/muzzy/database_adapters/mysql_adapter.rb

@@ -1,5 +1,6 @@
+require 'nkf'
 module Muzzy
-  module DatabaseAdapter
+  module DatabaseAdapters
     class MysqlAdapter < AdapterBase
       def initialize(config, verbose: false)
         @filepath = config[:filepath]
@@ -57,8 +58,8 @@ module Muzzy
       end
 
       # [Bool] true: table created, false: some error happened
-      def create_table(table_name, col_data_types)
-        create_table_sql = "CREATE TABLE #{table_name} (#{col_data_types.map{|x| "#{x.name} #{x.type}"}.join(', ')})"
+      def create_table(table_name, columns)
+        create_table_sql = "CREATE TABLE #{table_name} (#{columns.map{|x| "#{x.name} #{x.datatype}"}.join(', ')})"
         create_table_cmd_list = [*mysql_cmd_list, @database_name, '-e', '"', "#{create_table_sql}", '"']
         create_table_cmd = create_table_cmd_list.join(' ')
         if @verbose
@@ -73,6 +74,7 @@ module Muzzy
         cmds = [*mysqlimport_cmd_list, @database_name, '--local', @filepath]
         cmds.push "--ignore-lines=#{option[:first_row_is_header] ? 1 : 0}"
         cmds.push('--fields_enclosed_by="')
+        cmds.push("--lines-terminated-by=#{lines_terminated_by}")
 
         if option[:fields_terminated_by]
           cmds.push("--fields_terminated_by=#{option[:fields_terminated_by]}")
@@ -91,6 +93,17 @@ module Muzzy
 
       private
 
+      def lines_terminated_by
+        std_out, _ = Open3.capture2('file', @filepath)
+        if std_out =~ /with\sCR\s/
+          return '\r'
+        end
+        if std_out =~ /with\sCRLF\s/
+          return '\r\n'
+        end
+        return '\n'
+      end
+
       def mysql_cmd_list
         return @mysql_cmd_list if defined?(@mysql_cmd_list)
         @mysql_cmd_list = [@cmd_path] + common_param

data/lib/muzzy/header_detector.rb

@@ -5,7 +5,7 @@ module Muzzy
     # nil: could not detect
     def self.detect(rows)
       first_row, second_row = rows || []
-      return nil if first_row.empty?
+      return nil if first_row.nil? || first_row.empty?
 
       if first_row.any?{|str| str.to_s.match(/_id/i) }
         return true
@@ -15,15 +15,17 @@ module Muzzy
         return false
       end
 
-      return nil if second_row.empty?
+      return nil if second_row.nil? || second_row.empty?
 
       # I can't detect first_row is header or not, so guess now.
 
-      # general header row is not contain numbers
+      # header row is not contain numbers in most cases
       first_row_number_count = first_row.select{|str| str.to_f > 0}.length
-      return false if first_row_number_count > 0
+      if first_row_number_count > 0
+        return false
+      end
 
-      # If number col count is different, I guess first_row is header.
+      # If number col count is different, first_row is header.
       if first_row_number_count != second_row.select{|x| x.to_f > 0}.count
         return true
       end

data/lib/muzzy/util.rb

@@ -1,19 +1,29 @@
 require 'csv'
+require 'nkf'
 module Muzzy
   class Util
     def self.fetch_header_and_first_row(filepath, col_sep)
       raise ArgumentError, "filepath required"  if filepath.nil?
       raise ArgumentError, "not found file" unless File.exists?(filepath)
       header_row, first_row = nil, nil
-      CSV.foreach(filepath, col_sep: col_sep).each_with_index do |row, i|
-        if i == 0
-          header_row = row
-        elsif i == 1
-          first_row = row
-        else
-          break
+
+      from_fenc = 'UTF-8'
+      File.open(filepath, "rt") do |f|
+        s = f.readlines
+        from_fenc = NKF.guess(s.join).to_s
+      end
+      open(filepath, "rb:#{from_fenc}:UTF-8", undef: :replace) do |f|
+        CSV.new(f, col_sep: col_sep).each.with_index do |row, i|
+          if i == 0
+            header_row = row
+          elsif i == 1
+            first_row = row
+          else
+            break
+          end
         end
       end
+
       return [header_row, first_row]
     rescue ArgumentError => e
       raise e

data/lib/muzzy/version.rb

@@ -1,3 +1,3 @@
 module Muzzy
-  VERSION = "0.1.13"
+  VERSION = "0.1.14"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: muzzy
 version: !ruby/object:Gem::Version
-  version: 0.1.13
+  version: 0.1.14
 platform: ruby
 authors:
 - vimtaku
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-09-12 00:00:00.000000000 Z
+date: 2018-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -90,6 +90,8 @@ files:
 - exe/muzzy
 - exe/muzzy_setup
 - lib/muzzy.rb
+- lib/muzzy/column.rb
+- lib/muzzy/columns_generator.rb
 - lib/muzzy/database_adapters/adapter_base.rb
 - lib/muzzy/database_adapters/mysql_adapter.rb
 - lib/muzzy/filetype_detector.rb