rdb_csv 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f4dd0b5ba83e5aa0e4fa4a2e0f73b9471bc5f31a3eb5bbd6b2e7df073187e6c2
-  data.tar.gz: b18926dfa955cea6d3d4a7eaeaea53a1b4c97cc851e842770edf801552f4e6d7
+  metadata.gz: 49500956b9723214a17c58ea2511214c3b8d35f0734ed92609b14336e987cca6
+  data.tar.gz: 41896be81f877579013ce1d98410b413ad12b205e417817d6abd391f29a7feb2
 SHA512:
-  metadata.gz: 6ceb4b90f759b5d0060f018bc63da88a7f840b0cd9d67985d625b31469e9552a64a676fd2edf1e3daabaf9e678f8e854b4608a1f0c1baa77e9a45d6845384f3c
-  data.tar.gz: 16e8714188a2b7d7ff93c1fadb1345eb01906454107a2375107e512eca49c52713d12aa33ae554eb08182b5bf8d8a73325986dcc78ef6b8c4ae617100c74b16e
+  metadata.gz: fb447a0deab272668e546f3f067ad08957e2e7c6d1257b7fe76d22895a77e5f85ffb51ea61370ca3402f390b73555702ef1c8f24956ff7fed8e17ca5b9056e13
+  data.tar.gz: 4a9b3d811aa1ff10abded0c5e857c367cc50e9c28404d9138774ac47d5e50c86fc0021c8b9b0a3699057c8f7e1be51c23e7a781a8796116ac9714aaebd8ea1a0

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+vendor/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.6.3
+before_install: gem install bundler -v 2.0.1

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in rdb_csv.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,34 @@
+PATH
+  remote: .
+  specs:
+    rdb_csv (0.4.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.3)
+    rake (12.3.2)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.3)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rake
+  rdb_csv!
+  rspec
+
+BUNDLED WITH
+   2.0.1

data/LICENSE

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) 2019 longicorn
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,96 @@
+# RdbCSV
+
+RDB dumped csv/tsv can be read and write.
+
+Rdbcsv supoorts mainly MySQL and PostgreSQL within reasonable range.
+
+## Requirements
+- Ruby 2.4+
+
+## Installation
+
+```
+gem install rdb_csv
+```
+
+or, add this line in your Gemfile
+
+```
+gem 'rdb_csv'
+```
+
+## Usage
+
+`RdbCSV` is similar to the standard CSV class.
+
+database type: argment `db`. The default is to use standard CSV class.
+
+delimiter type: argment `delimiter` The default is to use `\t`, because the dafault delimiter for dump data is `\t`.
+
+### MySQL
+
+Use `INTO OUTFILE` to safely dump CSV,TSV on MySQL.
+
+```ruby
+require 'rdb_csv'
+
+# TSV
+tsv_path = "your_dump_file_path.tsv"
+RdbCSV.open(tsv_path, db: :mysql, delimiter: "\t") do |tsv|
+  tsv.each do |row|
+    p row
+  end
+end
+
+# CSV
+csv_path = "your_dump_file_path.tsv"
+RdbCSV.open(csv_path, db: :mysql, delimiter: ",") do |csv|
+  csv.each do |row|
+    p row
+  end
+end
+
+# Use foreach
+RdbCSV.foreach(csv_path, db: :mysql, delimiter: ",") do |row|
+  p row
+end
+```
+
+### PostgreSQl
+Use `COPY` to safely dump CSV,TSV on PostgreSQL.
+
+```ruby
+require 'rdb_csv'
+
+# TSV
+tsv_path = "your_dump_file_path.tsv"
+RdbCSV.foreach(tsv_path, db: :postgresql, delimiter: "\t") do |row|
+  p row
+end
+
+# CSV
+csv_path = "your_dump_file_path.csv"
+RdbCSV.foreach(csv_path, db: :postgresql, delimiter: ",") do |row|
+  p row
+end
+```
+
+### Convert MySQL to PostgreSQl
+
+```ruby
+require 'rdb_csv'
+
+mysql_rows = []
+
+mysql_tsv_path = "your_mysql_dump_file_path.tsv"
+RdbCSV.foreach(mysql_tsv_path, db: :mysql, delimiter: "\t") do |row|
+  mysql_rows << row
+end
+
+postgres_tsv_path = "your_postgres_dump_file_path.tsv"
+RdbCSV.open(postgres_tsv_path, "w", db: :postgresql, delimiter: "\t") do |tsv|
+  mysql_rows.each do |mysql_row|
+    tsv << mysql_row
+  end
+end
+```

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "rdb_csv"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/rdb_csv.rb

@@ -0,0 +1,123 @@
+require 'csv'
+require 'rdb_csv/reader'
+require 'rdb_csv/row'
+
+class RdbCSV
+  class CSV
+    include RdbCSVReader
+    include RdbCSVRow
+
+    def initialize(f, mode, options)
+      @f = f
+      @mode = options[:mode]
+      @db = options[:db]
+      @delimiter = options[:delimiter] || "\t"
+      @escape = "\\"
+      @linefeed = "\n"
+      quote = options[:quote] || '"'
+
+      # quote option is valid only mysql
+      @quote = options[:db] == :mysql ? quote : '"'
+    end
+
+    def each
+      raise IOError if @mode == 'w'
+
+      reader = Reader.new(@f, @db, @delimiter, escape: @escape, linefeed: @linefeed, quote: @quote)
+
+      reader.each_line do |row|
+        yield row.unescape(@escape, @db, @delimiter)
+      end
+    end
+
+    WRITE_BUFFER_SIZE = 1000
+
+    def <<(array)
+      raise IOError if @mode == 'r'
+
+      @lines ||= []
+
+      row = Row.new(array)
+      line = row.join(@escape, @db, @delimiter)
+      @lines << line + @linefeed
+      buffer_write if @lines.size >= WRITE_BUFFER_SIZE
+    end
+
+    def buffer_write
+      return if @lines.nil? || @lines.size.zero?
+
+      @f.write(@lines.join)
+      @lines = []
+    end
+  end
+
+  def self.open(file, mode = 'r', options)
+    db = options[:db] || :default
+    delimiter = options[:delimiter] || "\t"
+    quote = options[:delimiter] || '"'
+
+    case mode
+    when 'r'
+      if db == :default
+        ::CSV.open(file, mode, col_sep: delimiter) do |csv|
+          yield csv
+        end
+      else
+        File.open(file) do |f|
+          csv = CSV.new(f, mode, db: db, delimiter: delimiter, quote: quote)
+          yield csv
+        end
+      end
+    when 'w'
+      if db == :default
+        ::CSV.open(file, mode, col_sep: delimiter) do |csv|
+          yield csv
+        end
+      else
+        File.open(file, 'w') do |f|
+          csv = CSV.new(f, mode, db: db, delimiter: delimiter, quote: quote)
+          yield csv
+
+          csv.buffer_write
+        end
+      end
+    end
+  end
+
+  def self.foreach(file, options)
+    open(file, options) do |csv|
+      csv.each do |row|
+        yield row
+      end
+    end
+  end
+
+  def self.parse(str, options)
+    db = options[:db] || :default
+    delimiter = options[:delimiter] || "\t"
+    quote = options[:delimiter] || '"'
+
+    if options.key?(:db)
+      reader = CSV.new(str, 'r', db: db, delimiter: delimiter, quote: quote)
+    else
+      reader = ::CSV.parse(str, col_sep: delimiter)
+    end
+
+    rows = []
+    reader.each do |row|
+      if block_given?
+        yield row
+      else
+        rows << row
+      end
+    end
+
+    return rows unless block_given?
+  end
+
+  def self.parse_line(line, options)
+    ary = nil
+    parse(line, options).each{|row| ary = row; break}
+    return ary
+  end
+end

data/lib/rdb_csv/reader.rb

@@ -0,0 +1,73 @@
+module RdbCSVReader
+  class Reader
+    def initialize(fp, db, delimiter, options)
+      @fp = fp
+      @db = db
+      @delimiter = delimiter
+      @escape = options[:escape] || "\\" # 1 char
+      @linefeed = options[:linefeed] || "\n"
+      @quote = options[:quote] || '"'
+    end
+
+    def each_line
+      str = ''
+      row = []
+      quote_num = 0
+
+      each_char do |char|
+        if quote_num > 0
+          if str[-1] == @quote && char == @quote
+            quote_num -= 1
+          elsif str[-1] != @quote && char == @quote
+            quote_num -= 1
+          elsif char == @quote
+            quote_num += 1
+          end
+          str += char
+        else
+          if char == @delimiter
+            row << str
+            str = ''
+          elsif char == @linefeed
+            row << str
+            yield RdbCSV::CSV::Row.new(row)
+            row = []
+            str = ''
+          else
+            quote_num += 1 if (str.size == 0 || str[0] == @quote) && char == @quote
+            str += char
+          end
+        end
+      end
+    end
+
+    private
+
+    def each_char
+      str = ''
+      @fp.each_char do |char|
+        if str[-1] == @escape
+          str += char
+          yield str
+          str = ''
+          next
+        end
+
+        str += char
+
+        if str[@linefeed.size-1..-1] == @linefeed
+          yield str
+          str = ''
+          next
+        end
+
+        if str[-1] != @escape
+          yield str
+          str = ''
+        end
+      end
+
+      yield str unless str.empty?
+    end
+  end
+end

data/lib/rdb_csv/row.rb

@@ -0,0 +1,121 @@
+module RdbCSVRow
+  class Row < Array
+    def unescape(escape_char, db, delimiter)
+      send("unescape_#{db}").map{|v|v.to_i.to_s == v ? v.to_i : v rescue v}
+    end
+
+    def join(escape_char, db, delimiter)
+      escape(escape_char, db, delimiter).to_a.join(delimiter)
+    end
+
+    private
+
+    def unescape_default
+      self
+    end
+
+    def unescape_mysql
+      row = self
+      if @quote
+        row = row.map{|v|v.match?(/^#{@quote}.*?#{@quote}$/m) ? v[1..-2].gsub(/#{@quote}#{@quote}/o, @quote) : v}
+      end
+      row.map{|v|v.nil? ? nil : v.gsub(/\\\\/o, '\\')
+                                  .gsub(/\\n/o, "\n")
+                                  .gsub(/\\\n/o, "\n")
+                                  .gsub(/\\r/o, "\r")
+                                  .gsub(/\\t/o, "\t")
+                                  .gsub(/\\0/o, "\0")
+                                  .gsub(/\\,/o, ",")
+              }
+          .map{|v|v=="\\N" ? nil : v}
+          .map{|v|v=="NULL" ? nil : v}
+    end
+
+    def unescape_postgresql
+      self.map{|v|v.gsub(/\\0/o, "\0")}
+          .map{|v|v.match?(/^".*?"$/om) ? v[1..-2].gsub(/""/, '"') : (v == '' ? nil : v)}
+    end
+
+    def escape(escape_char, db, delimiter)
+      send("escape_#{db}", escape_char, delimiter)
+    end
+
+    def escape_default(escape_char, delimiter)
+      self
+    end
+
+    def escape_mysql(escape_char, delimiter)
+      case delimiter
+      when "\t" # tsv
+        self.map do |column|
+          case column
+          when nil
+            "NULL"
+          when String
+            str = ""
+            column.each_char do |char|
+              case char
+              when escape_char
+                str += char*2
+              when "\n"
+                str += "#{escape_char}n"
+              when "\t"
+                str += "#{escape_char}t"
+              when "\0"
+                str += "#{escape_char}0"
+              else
+                str += char
+              end
+            end
+            str
+          else
+            column
+          end
+        end
+      when "," # csv
+        self.map do |column|
+          case column
+          when nil
+            "#{escape_char}N"
+          when String
+            str = ""
+            column.each_char do |char|
+              case char
+              when escape_char
+                str += char*2
+              when "\n"
+                str += "#{escape_char}\n"
+              when "\0"
+                str += "#{escape_char}0"
+              when delimiter
+                str += "#{escape_char}#{delimiter}"
+              else
+                str += char
+              end
+            end
+            str
+          else
+            column
+          end
+        end
+      end
+    end
+
+    def escape_postgresql(escape_char, delimiter)
+      self.map do |v|
+        if v == ''
+          '""'
+        elsif v.nil?
+          ''
+        elsif v.kind_of?(String)
+          v = v.gsub(/\0/o, "#{escape_char}#{escape_char}0")
+          v = v.gsub(/"/o, '""') if v.include?('"')
+          v = "\"#{v}\"" if v.match?(/"|\r|\n|#{delimiter}/)
+          v
+        else
+          v
+        end
+      end
+    end
+  end
+end

data/lib/rdb_csv/version.rb

@@ -0,0 +1,3 @@
+class RdbCSV
+  VERSION = "0.4.0"
+end

data/rdb_csv.gemspec

@@ -0,0 +1,21 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "rdb_csv/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "rdb_csv"
+  spec.version       = RdbCSV::VERSION
+  spec.authors       = ["longicorn"]
+  spec.email         = ["longicorn.c@gmail.com"]
+
+  spec.summary       = %q{RDB dumped csv/tsv can be read and write.}
+  spec.description   = %q{Rdbcsv supoorts mainly MySQL and PostgreSQL within reasonable range.}
+  spec.homepage      = "https://github.com/longicorn/rdb_csv"
+  spec.license       = 'MIT'
+
+  spec.require_paths = ["lib"]
+  spec.files         = `git ls-files`.split($/)
+
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rake"
+end

data/spec/fixtures/mysql.csv

@@ -0,0 +1,7 @@
+1,abcdefg,ABCNDEFG,1,1,2018-12-12 02:26:06,2018-12-12 02:26:06,2018-12-12 02:26:06
+2,abc\
+
+3,abc	defg,ABC\,DEFG,\N,-1,2018-12-12 02:26:06,2018-12-12 02:26:06,2018-12-12 02:26:06
+4,abc\\defg,ABC"DEFG,\N,-1,2018-12-12 02:26:06,2018-12-12 02:26:06,2018-12-12 02:26:06
+5,'abcdefg,ABC'DEFG,\N,-1,2018-12-12 02:26:06,2018-12-12 02:26:06,2018-12-12 02:26:06
+6,\N,,\N,\N,2018-12-12 02:26:06,2018-12-12 02:26:06,2018-12-12 02:26:06

data/spec/fixtures/mysql.tsv

@@ -0,0 +1,6 @@
+1	abcdefg	ABCNDEFG	1	1	2018-12-12 02:26:06	2018-12-12 02:26:06	2018-12-12 02:26:06
+2	abc\n
+3	abc\tdefg	ABC,DEFG	NULL	-1	2018-12-12 02:26:06	2018-12-12 02:26:06	2018-12-12 02:26:06
+4	abc\\defg	ABC"DEFG	NULL	-1	2018-12-12 02:26:06	2018-12-12 02:26:06	2018-12-12 02:26:06
+5	'abcdefg	ABC'DEFG	NULL	-1	2018-12-12 02:26:06	2018-12-12 02:26:06	2018-12-12 02:26:06
+6	NULL		NULL	NULL	2018-12-12 02:26:06	2018-12-12 02:26:06	2018-12-12 02:26:06

data/spec/fixtures/normal.csv

@@ -0,0 +1,4 @@
+abcdefg,hijkmnl,opqrstu,vwxyz
+あいうえお,かきくけこ,さしすせそ,たちつてと
+"あいうえお
+かきくけこ",さしすせそ,たちつてと

data/spec/fixtures/normal.tsv

@@ -0,0 +1,4 @@
+abcdefg	hijkmnl	opqrstu	vwxyz
+あいうえお	かきくけこ	さしすせそ	たちつてと
+"あいうえお
+かきくけこ"	さしすせそ	たちつてと

data/spec/fixtures/postgresql.csv

@@ -0,0 +1,8 @@
+id,text0,text1,flag,point,inserted_at,created_at,updated_at
+1,abcdefg,ABCNDEFG,t,1,2018-12-18 04:18:24.181256,2018-12-18 04:18:24.18319,2018-12-18 04:18:24.18319
+2,"abc
+
+3,abc	defg,"ABC,DEFG",,-1,2018-12-18 04:18:24.202638,2018-12-18 04:18:24.203768,2018-12-18 04:18:24.203768
+4,abc\defg,"ABC""DEFG",,-1,2018-12-18 04:18:24.207705,2018-12-18 04:18:24.20904,2018-12-18 04:18:24.20904
+5,'abcdefg,ABC'DEFG,,-1,2018-12-18 04:18:24.207705,2018-12-18 04:18:24.20904,2018-12-18 04:18:24.20904
+6,,"",,,2018-12-18 04:18:24.213643,2018-12-18 04:18:24.215238,2018-12-18 04:18:24.215238

data/spec/fixtures/postgresql.tsv

@@ -0,0 +1,8 @@
+id	text0	text1	flag	point	inserted_at	created_at	updated_at
+1	abcdefg	ABCNDEFG	t	1	2018-12-18 04:18:24.181256	2018-12-18 04:18:24.18319	2018-12-18 04:18:24.18319
+2	"abc
+
+3	"abc	defg"	ABC,DEFG		-1	2018-12-18 04:18:24.202638	2018-12-18 04:18:24.203768	2018-12-18 04:18:24.203768
+4	abc\defg	"ABC""DEFG"		-1	2018-12-18 04:18:24.207705	2018-12-18 04:18:24.20904	2018-12-18 04:18:24.20904
+5	'abcdefg	ABC'DEFG		-1	2018-12-18 04:18:24.207705	2018-12-18 04:18:24.20904	2018-12-18 04:18:24.20904
+6		""			2018-12-18 04:18:24.213643	2018-12-18 04:18:24.215238	2018-12-18 04:18:24.215238

data/spec/load/README.md

@@ -0,0 +1,37 @@
+# Load test
+
+## MySQL
+```
+$ cp spec/load/docker/docker-compose.mysql.yml docker-compose.yml
+$ sudo docker-compose up -d
+$ mysql -h 127.0.0.1 -P 3306 -u docker -p < spec/load/create_table.mysql.sql
+```
+
+load csv
+```
+$ mysql -h 127.0.0.1 -P 3306 -u docker -p < spec/load/load.mysql.csv.sql
+```
+
+load tsv
+```
+$ mysql -h 127.0.0.1 -P 3306 -u docker -p < spec/load/load.mysql.tsv.sql
+```
+
+## PostgreSQL
+```
+$ cp spec/load/docker/docker-compose.postgresql.yml docker-compose.yml
+$ sudo docker-compose up -d
+$ mysql -h 127.0.0.1 -P 3306 -u docker -p < spec/load/create_table.postgresql.sql
+```
+
+load csv
+```
+$ sudo docker cp spec/fixtures/postgresql.csv <CONTAINER ID>:/tmp/postgresql.csv
+$ psql -h 127.0.0.1 -p 5432 -U docker < spec/load/load.postgresql.csv.sql
+```
+
+load tsv
+```
+$ sudo docker cp spec/fixtures/postgresql.tsv <CONTAINER ID>:/tmp/postgresql.csv
+$ psql -h 127.0.0.1 -p 5432 -U docker < spec/load/load.postgresql.tsv.sql
+```

data/spec/load/create_table.mysql.sql

@@ -0,0 +1 @@
+create table test (id int, text0 varchar(255), text1 varchar(255), flag tinyint(1), point int, inserted_at datetime, created_at datetime, updated_at datetime) CHARSET=utf8mb4;

data/spec/load/create_table.postgresql.sql

@@ -0,0 +1 @@
+create table test (id int, text0 varchar(255), text1 varchar(255), flag boolean, point int, inserted_at timestamp, created_at timestamp, updated_at timestamp);

data/spec/load/docker/docker-compose.mysql.yml

@@ -0,0 +1,16 @@
+version: '3'
+
+services:
+  db:
+    image: mysql:5.7
+    container_name: rdb_csv_mysql
+    environment:
+      MYSQL_ROOT_PASSWORD: root
+      MYSQL_DATABASE: test
+      MYSQL_USER: docker
+      MYSQL_PASSWORD: password
+    command:
+      - --character-set-server=utf8mb4
+      - --collation-server=utf8mb4_unicode_ci
+    ports:
+      - 3306:3306

data/spec/load/docker/docker-compose.postgresql.yml

@@ -0,0 +1,10 @@
+version: '3'
+
+services:
+  postgres:
+    image: postgres
+    environment:
+      POSTGRES_USER: docker
+      POSTGRES_PASSWORD: password
+    ports:
+      - -5432:5432

data/spec/load/load.mysql.csv.sql

@@ -0,0 +1 @@
+LOAD DATA LOCAL INFILE "./spec/fixtures/mysql.csv" INTO TABLE test FIELDS TERMINATED BY ',';

data/spec/load/load.mysql.tsv.sql

@@ -0,0 +1 @@
+LOAD DATA LOCAL INFILE "./spec/fixtures/mysql.tsv" INTO TABLE test FIELDS TERMINATED BY '\t';

data/spec/load/load.postgresql.csv.sql

@@ -0,0 +1 @@
+COPY test FROM '/tmp/postgresql.csv' WITH CSV HEADER;

data/spec/load/load.postgresql.tsv.sql

@@ -0,0 +1 @@
+COPY test FROM '/tmp/postgresql.tsv' WITH CSV HEADER DELIMITER E'\t';

data/spec/rdb_csv_spec.rb

@@ -0,0 +1,512 @@
+require 'fileutils'
+require 'csv'
+
+RSpec.describe RdbCSV do
+  before do
+    FileUtils.mkdir_p("tmp")
+  end
+
+  after do
+    FileUtils.rm_rf("tmp")
+  end
+
+  it "Check version number" do
+    expect(RdbCSV::VERSION).to eq "0.4.0"
+  end
+
+  describe "normal data" do
+    let(:check_rows) {[["abcdefg","hijkmnl","opqrstu","vwxyz"],
+                       ["あいうえお","かきくけこ","さしすせそ","たちつてと"],
+                       ["あいうえお\nかきくけこ","さしすせそ","たちつてと"]
+                      ]}
+
+    describe "Read csv" do
+      it "open" do
+        read_rows = []
+        path = "spec/fixtures/normal.csv"
+
+        RdbCSV.open(path, delimiter: ",") do |csv|
+          csv.each do |row|
+            read_rows << row
+          end
+        end
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "foreach" do
+        read_rows = []
+        path = "spec/fixtures/normal.csv"
+
+        RdbCSV.foreach(path, delimiter: ",") do |row|
+          read_rows << row
+        end
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "parse without block" do
+        read_rows = []
+        path = "spec/fixtures/normal.csv"
+
+        str = File.read(path)
+        read_rows = RdbCSV.parse(str, delimiter: ",")
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "parse with block" do
+        read_rows = []
+        path = "spec/fixtures/normal.csv"
+
+        str = File.read(path)
+        RdbCSV.parse(str, delimiter: ",").each do |row|
+          read_rows << row
+        end
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "parse_line" do
+        read_rows = []
+        path = "spec/fixtures/normal.csv"
+
+        str = File.read(path)
+        read_rows = RdbCSV.parse_line(str, delimiter: ",")
+
+        expect(read_rows).to eq check_rows[0]
+      end
+    end
+
+    describe "Read tsv" do
+      it "open" do
+        read_rows = []
+        path = "spec/fixtures/normal.tsv"
+
+        RdbCSV.open(path, delimiter: "\t") do |tsv|
+          tsv.each do |row|
+            read_rows << row
+          end
+        end
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "foreach" do
+        read_rows = []
+        path = "spec/fixtures/normal.tsv"
+
+        RdbCSV.foreach(path, delimiter: "\t") do |row|
+          read_rows << row
+        end
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "parse without block" do
+        read_rows = []
+        path = "spec/fixtures/normal.tsv"
+
+        str = File.read(path)
+        read_rows = RdbCSV.parse(str, delimiter: "\t")
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "parse with block" do
+        read_rows = []
+        path = "spec/fixtures/normal.tsv"
+
+        str = File.read(path)
+        RdbCSV.parse(str, delimiter: "\t").each do |row|
+          read_rows << row
+        end
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "parse_line" do
+        read_rows = []
+        path = "spec/fixtures/normal.tsv"
+
+        str = File.read(path)
+        read_rows = RdbCSV.parse_line(str, delimiter: "\t")
+
+        expect(read_rows).to eq check_rows[0]
+      end
+    end
+
+    it "Write csv" do
+      read_rows = []
+      path = "tmp/normal.csv"
+
+      RdbCSV.open(path, 'w', delimiter: ",") do |csv|
+        check_rows.each do |row|
+          csv << row
+        end
+      end
+
+      RdbCSV.open(path, delimiter: ",") do |csv|
+        csv.each do |row|
+          read_rows << row
+        end
+      end
+
+      expect(read_rows).to eq check_rows
+    end
+
+    it "Write csv" do
+      read_rows = []
+      path = "tmp/normal.csv"
+
+      RdbCSV.open(path, 'w', delimiter: "\t") do |csv|
+        check_rows.each do |row|
+          csv << row
+        end
+      end
+
+      RdbCSV.open(path, delimiter: "\t") do |csv|
+        csv.each do |row|
+          read_rows << row
+        end
+      end
+
+      expect(read_rows).to eq check_rows
+    end
+  end
+
+  describe "MySQL data" do
+    let(:check_rows) {[[1, "abcdefg", "ABCNDEFG", 1, 1, "2018-12-12 02:26:06", "2018-12-12 02:26:06", "2018-12-12 02:26:06"],
+                       [2, "abc\n\rdefg", "ABC\u0000DEFG", 0, 0, "2018-12-12 02:26:06", "2018-12-12 02:26:06", "2018-12-12 02:26:06"],
+                       [3, "abc\tdefg", "ABC,DEFG", nil, -1, "2018-12-12 02:26:06", "2018-12-12 02:26:06", "2018-12-12 02:26:06"],
+                       [4, "abc\\defg", "ABC\"DEFG", nil, -1, "2018-12-12 02:26:06", "2018-12-12 02:26:06", "2018-12-12 02:26:06"],
+                       [5, "'abcdefg", "ABC'DEFG", nil, -1, "2018-12-12 02:26:06", "2018-12-12 02:26:06", "2018-12-12 02:26:06"],
+                       [6, nil, "", nil, nil, "2018-12-12 02:26:06", "2018-12-12 02:26:06", "2018-12-12 02:26:06"]
+                      ]}
+
+    describe "Read csv" do
+      it "open" do
+        read_rows = []
+        path = "spec/fixtures/mysql.csv"
+
+        RdbCSV.open(path, db: :mysql, delimiter: ",") do |csv|
+          csv.each do |row|
+            read_rows << row
+          end
+        end
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "foreach" do
+        read_rows = []
+        path = "spec/fixtures/mysql.csv"
+
+        RdbCSV.foreach(path, db: :mysql, delimiter: ",") do |row|
+          read_rows << row
+        end
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "parse without block" do
+        read_rows = []
+        path = "spec/fixtures/mysql.csv"
+
+        str = File.read(path)
+        read_rows = RdbCSV.parse(str, db: :mysql, delimiter: ",")
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "parse with block" do
+        read_rows = []
+        path = "spec/fixtures/mysql.csv"
+
+        str = File.read(path)
+        RdbCSV.parse(str, db: :mysql, delimiter: ",").each do |row|
+          read_rows << row
+        end
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "parse_line" do
+        read_rows = []
+        path = "spec/fixtures/mysql.csv"
+
+        str = File.read(path)
+        read_rows = RdbCSV.parse_line(str, db: :mysql, delimiter: ",")
+
+        expect(read_rows).to eq check_rows[0]
+      end
+    end
+
+    describe "Read tsv" do
+      it "open" do
+        read_rows = []
+        path = "spec/fixtures/mysql.tsv"
+
+        RdbCSV.open(path, db: :mysql, delimiter: "\t") do |tsv|
+          tsv.each do |row|
+            read_rows << row
+          end
+        end
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "foreach" do
+        read_rows = []
+        path = "spec/fixtures/mysql.tsv"
+
+        RdbCSV.foreach(path, db: :mysql, delimiter: "\t") do |row|
+          read_rows << row
+        end
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "parse without block" do
+        read_rows = []
+        path = "spec/fixtures/mysql.tsv"
+
+        str = File.read(path)
+        read_rows = RdbCSV.parse(str, db: :mysql, delimiter: "\t")
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "parse with block" do
+        read_rows = []
+        path = "spec/fixtures/mysql.tsv"
+
+        str = File.read(path)
+        RdbCSV.parse(str, db: :mysql, delimiter: "\t").each do |row|
+          read_rows << row
+        end
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "parse_line" do
+        read_rows = []
+        path = "spec/fixtures/mysql.tsv"
+
+        str = File.read(path)
+        read_rows = RdbCSV.parse_line(str, db: :mysql, delimiter: "\t")
+
+        expect(read_rows).to eq check_rows[0]
+      end
+    end
+
+    it "Write csv" do
+      read_rows = []
+      path = "tmp/mysql.csv"
+
+      RdbCSV.open(path, 'w', db: :mysql, delimiter: ",") do |csv|
+        check_rows.each do |row|
+          csv << row
+        end
+      end
+
+      RdbCSV.open(path, db: :mysql, delimiter: ",") do |csv|
+        csv.each do |row|
+          read_rows << row
+        end
+      end
+
+      expect(read_rows).to eq check_rows
+    end
+
+    it "Write tsv" do
+      read_rows = []
+      path = "tmp/mysql.tsv"
+
+      RdbCSV.open(path, 'w', db: :mysql, delimiter: "\t") do |tsv|
+        check_rows.each do |row|
+          tsv << row
+        end
+      end
+
+      RdbCSV.open(path, db: :mysql, delimiter: "\t") do |tsv|
+        tsv.each do |row|
+          read_rows << row
+        end
+      end
+
+      expect(read_rows).to eq check_rows
+    end
+  end
+
+  describe "PostgreSQL data" do
+    let(:check_rows) {[["id", "text0", "text1", "flag", "point", "inserted_at", "created_at", "updated_at"],
+                       [1, "abcdefg", "ABCNDEFG", "t", 1, "2018-12-18 04:18:24.181256",
+                        "2018-12-18 04:18:24.18319", "2018-12-18 04:18:24.18319"],
+                       [2, "abc\n\rdefg", "ABC\u0000DEFG", "f", 0,
+                        "2018-12-18 04:18:24.19734", "2018-12-18 04:18:24.198651", "2018-12-18 04:18:24.198651"],
+                       [3, "abc\tdefg", "ABC,DEFG", nil, -1,
+                        "2018-12-18 04:18:24.202638", "2018-12-18 04:18:24.203768", "2018-12-18 04:18:24.203768"],
+                       [4, "abc\\defg", "ABC\"DEFG", nil, -1,
+                        "2018-12-18 04:18:24.207705", "2018-12-18 04:18:24.20904", "2018-12-18 04:18:24.20904"],
+                       [5, "'abcdefg", "ABC'DEFG", nil, -1,
+                        "2018-12-18 04:18:24.207705", "2018-12-18 04:18:24.20904", "2018-12-18 04:18:24.20904"],
+                       [6, nil, "", nil, nil,
+                        "2018-12-18 04:18:24.213643", "2018-12-18 04:18:24.215238", "2018-12-18 04:18:24.215238"]
+                      ]}
+
+    describe "Read csv" do
+      it "open" do
+        read_rows = []
+        path = "spec/fixtures/postgresql.csv"
+
+        RdbCSV.open(path, db: :postgresql, delimiter: ",") do |csv|
+          csv.each do |row|
+            read_rows << row
+          end
+        end
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "foreach" do
+        read_rows = []
+        path = "spec/fixtures/postgresql.csv"
+
+        RdbCSV.foreach(path, db: :postgresql, delimiter: ",") do |row|
+          read_rows << row
+        end
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "parse without block" do
+        read_rows = []
+        path = "spec/fixtures/postgresql.csv"
+
+        str = File.read(path)
+        read_rows = RdbCSV.parse(str, db: :postgresql, delimiter: ",")
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "parse with block" do
+        read_rows = []
+        path = "spec/fixtures/postgresql.csv"
+
+        str = File.read(path)
+        RdbCSV.parse(str, db: :postgresql, delimiter: ",").each do |row|
+          read_rows << row
+        end
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "parse_line" do
+        read_rows = []
+        path = "spec/fixtures/postgresql.csv"
+
+        str = File.read(path)
+        read_rows = RdbCSV.parse_line(str, db: :postgresql, delimiter: ",")
+
+        expect(read_rows).to eq check_rows[0]
+      end
+    end
+
+    describe "Read tsv" do
+      it "open" do
+        read_rows = []
+        path = "spec/fixtures/postgresql.tsv"
+
+        RdbCSV.open(path, db: :postgresql, delimiter: "\t") do |tsv|
+          tsv.each do |row|
+            read_rows << row
+          end
+        end
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "foreach" do
+        read_rows = []
+        path = "spec/fixtures/postgresql.tsv"
+
+        RdbCSV.foreach(path, db: :postgresql, delimiter: "\t") do |row|
+          read_rows << row
+        end
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "parse without block" do
+        read_rows = []
+        path = "spec/fixtures/postgresql.tsv"
+
+        str = File.read(path)
+        read_rows = RdbCSV.parse(str, db: :postgresql, delimiter: "\t")
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "parse with block" do
+        read_rows = []
+        path = "spec/fixtures/postgresql.tsv"
+
+        str = File.read(path)
+        RdbCSV.parse(str, db: :postgresql, delimiter: "\t").each do |row|
+          read_rows << row
+        end
+
+        expect(read_rows).to eq check_rows
+      end
+
+      it "parse_line" do
+        read_rows = []
+        path = "spec/fixtures/postgresql.tsv"
+
+        str = File.read(path)
+        read_rows = RdbCSV.parse_line(str, db: :postgresql, delimiter: "\t")
+
+        expect(read_rows).to eq check_rows[0]
+      end
+    end
+
+    it "Write csv" do
+      read_rows = []
+      path = "tmp/postgresql.csv"
+
+      RdbCSV.open(path, 'w', db: :postgresql, delimiter: ",") do |csv|
+        check_rows.each do |row|
+          csv << row
+        end
+      end
+
+      RdbCSV.open(path, db: :postgresql, delimiter: ",") do |csv|
+        csv.each do |row|
+          read_rows << row
+        end
+      end
+
+      expect(read_rows).to eq check_rows
+    end
+
+    it "Write tsv" do
+      read_rows = []
+      path = "tmp/postgresql.tsv"
+
+      RdbCSV.open(path, 'w', db: :postgresql, delimiter: ",") do |tsv|
+        check_rows.each do |row|
+          tsv << row
+        end
+      end
+
+      RdbCSV.open(path, db: :postgresql, delimiter: ",") do |tsv|
+        tsv.each do |row|
+          read_rows << row
+        end
+      end
+
+      expect(read_rows).to eq check_rows
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,14 @@
+require "bundler/setup"
+require_relative "../lib/rdb_csv.rb"
+
+RSpec.configure do |config|
+  # Enable flags like --only-failures and --next-failure
+  config.example_status_persistence_file_path = ".rspec_status"
+
+  # Disable RSpec exposing methods globally on `Module` and `main`
+  config.disable_monkey_patching!
+
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdb_csv
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - longicorn
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-26 00:00:00.000000000 Z
+date: 2019-08-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -44,7 +44,39 @@ email:
 executables: []
 extensions: []
 extra_rdoc_files: []
-files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/rdb_csv.rb
+- lib/rdb_csv/reader.rb
+- lib/rdb_csv/row.rb
+- lib/rdb_csv/version.rb
+- rdb_csv.gemspec
+- spec/fixtures/mysql.csv
+- spec/fixtures/mysql.tsv
+- spec/fixtures/normal.csv
+- spec/fixtures/normal.tsv
+- spec/fixtures/postgresql.csv
+- spec/fixtures/postgresql.tsv
+- spec/load/README.md
+- spec/load/create_table.mysql.sql
+- spec/load/create_table.postgresql.sql
+- spec/load/docker/docker-compose.mysql.yml
+- spec/load/docker/docker-compose.postgresql.yml
+- spec/load/load.mysql.csv.sql
+- spec/load/load.mysql.tsv.sql
+- spec/load/load.postgresql.csv.sql
+- spec/load/load.postgresql.tsv.sql
+- spec/rdb_csv_spec.rb
+- spec/spec_helper.rb
 homepage: https://github.com/longicorn/rdb_csv
 licenses:
 - MIT