rabbit-slide-kou-rubykaigi-2019 2019.4.19.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f4d26666f0ff8c0fbd0ea010f55853e41383a03bda2d9c8909d6eabd80d19c77
+  data.tar.gz: eaf69441d728a122d8f937610381d126919023e05c9fe3010f061e2d8bbbd851
+SHA512:
+  metadata.gz: a8665f7ce21d4bd9bc6790a84f46505f0913ad919aba46a2a4651672216c56a855b9e486d99ddaeceea30646772e6f6e07ae276685d15d16a4936777c15bec51
+  data.tar.gz: f7f3bf3aaaee69d9857c4070575dfc6d0eb5cc96a52d42887a80aa0fcb8cb9e5e752984eddd9a8dadb8704e0c3243221fb70d5aba4a86f919950a974339d02f7

data/.rabbit

@@ -0,0 +1 @@
+--size=800,450 csv.rab

data/README.rd

@@ -0,0 +1,52 @@
+= Better CSV processing with Ruby 2.6
+
+csv, one of the standard libraries, in Ruby 2.6 has many improvements:
+
+  * Default gemified
+  * Faster CSV parsing
+  * Faster CSV writing
+  * Clean new CSV parser implementation for further improvements
+  * Reconstructed test suites for further improvements
+  * Benchmark suites for further performance improvements
+
+These improvements are done without breaking backward compatibility.
+
+This talk describes details of these improvements by a new csv maintainer.
+
+== License
+
+=== Slide
+
+CC BY-SA 4.0
+
+Use the followings for notation of the author:
+
+  * Kouhei Sutou
+
+==== ClearCode Inc. logo
+
+CC BY-SA 4.0
+
+Author: ClearCode Inc.
+
+It is used in page header and some pages in the slide.
+
+== For author
+
+=== Show
+
+  rake
+
+=== Publish
+
+  rake publish
+
+== For viewers
+
+=== Install
+
+  gem install rabbit-slide-kou-rubykaigi-2019
+
+=== Show
+
+  rabbit rabbit-slide-kou-rubykaigi-2019.gem

data/Rakefile

@@ -0,0 +1,32 @@
+require "rabbit/task/slide"
+
+# Edit ./config.yaml to customize meta data
+
+spec = nil
+Rabbit::Task::Slide.new do |task|
+  spec = task.spec
+  # spec.files += Dir.glob("doc/**/*.*")
+  # spec.files -= Dir.glob("private/**/*.*")
+  spec.add_runtime_dependency("rabbit-theme-clear-code")
+end
+
+desc "Tag #{spec.version}"
+task :tag do
+  sh("git", "tag", "-a", spec.version.to_s, "-m", "Publish #{spec.version}")
+  sh("git", "push", "--tags")
+end
+
+desc "Run benchmark"
+task :benchmark do
+  env = {
+    "RUBYLIB" => nil,
+    "BUNDLER_ORIG_RUBYLIB" => nil,
+  }
+  sh(env,
+     FileUtils::RUBY,
+     "-S",
+     "benchmark-driver",
+     "--repeat-count", "1",
+     "--runner", "time",
+     "benchmark.yaml")
+end

data/config.yaml

@@ -0,0 +1,23 @@
+---
+id: rubykaigi-2019
+base_name: csv
+tags:
+- rabbit
+- csv
+- apachearrow
+presentation_date: 2019-04-19
+version: 2019.4.19.0
+licenses:
+- CC-BY-SA-4.0
+slideshare_id: 
+speaker_deck_id: 
+ustream_id: 
+vimeo_id: 
+youtube_id: 
+author:
+  markup_language: :rd
+  name: Kouhei Sutou
+  email: kou@clear-code.com
+  rubygems_user: kou
+  slideshare_user: kou
+  speaker_deck_user: 

data/csv.rab

@@ -0,0 +1,392 @@
+= Better CSV processing\nwith Ruby 2.6
+
+: author
+   Kouhei Sutou
+: institution
+   ClearCode Inc.
+: content-source
+   RubyKaigi 2019
+: date
+   2019-04-19
+: start-time
+   2019-04-19T11:20:00+09:00
+: end-time
+   2019-04-19T12:00:00+09:00
+: theme
+   .
+
+= Ad: Silver sponsor
+
+  # img
+  # src = images/clear-code-rubykaigi-2019-silver-sponsor.png
+  # relative_height = 100
+  # reflect_ratio = 0.1
+
+== Slide properties
+
+: enable-title-on-image
+   false
+
+= Me(('note:（自己紹介）'))
+
+  * The president of ClearCode Inc.\n
+    (('note:クリアコードの社長'))
+  * A new maintainer of the (({csv})) library\n
+    (('note:(({csv}))ライブラリーの新メンテナー'))
+  * The founder of ((<Red Data Tools|URL:https://red-data-tools.github.io/>)) project:\n
+    (('note:Red Data Toolsプロジェクトの立ち上げ人'))
+    * Provides data processing tools for Ruby\n
+      (('note:Ruby用のデータ処理ツールを提供するプロジェクト'))
+
+= Kazuma Furuhashi
+
+  * ...
+
+= CSV in Ruby 2.6 (1)\n(('note:Ruby 2.6のCSV (1)'))
+
+Faster CSV parsing\n
+(('note:CSVパースの高速化'))
+
+= Unquoted CSV\n(('note:クォートなしのCSV'))
+
+    AAAAA,AAAAA,AAAAA
+    ...
+
+  # RT
+
+  2.5, 2.6, Faster?
+
+  432.0i/s, 764.9i/s, 1.77x
+
+= Quoted CSV\n(('note:クォートありのCSV'))
+
+    "AAAAA","AAAAA","AAAAA"
+    ...
+
+  # RT
+
+  2.5, 2.6, Faster?
+
+  274.1i/s, 534.5i/s, 1.95x
+
+= Quoted separator CSV (1)\n(('note:区切り文字をクォートしているCSV (1)'))
+
+    ",AAAAA",",AAAAA",",AAAAA"
+    ...
+
+  # RT
+
+  2.5, 2.6, Faster?
+
+  211.0i/s, 330.0/s, 1.56x
+
+= Quoted separator CSV (2)\n(('note:区切り文字をクォートしているCSV (2)'))
+
+    "AAAAA\r\n","AAAAA\r\n","AAAAA\r\n"
+    ...
+
+  # RT
+
+  2.5, 2.6, Faster?
+
+  118.7i/s, 325.6/s, 2.74x
+
+= Quoted CSVs\n(('note:クォートありのCSV'))
+
+  # RT
+
+  , 2.5, 2.6
+
+  Just quoted, 274.1i/s, 554.5i/s
+  Include sep1, 211.0i/s, 330.0i/s
+  Include sep2, 118.0i/s, 325.6i/s
+  (Note), (Slow down), (Still fast)
+
+= Multibyte CSV\n(('note:マルチバイトのCSV'))
+
+    あああああ,あああああ,あああああ
+    ...
+
+  # RT
+
+  2.5, 2.6, Faster?
+
+  371.2i/s, 626.6i/s, 1.69x
+
+= CSV in Ruby 2.6 (2)\n(('note:Ruby 2.6のCSV (1)'))
+
+Faster CSV writing\n
+(('note:CSV書き出しの高速化'))
+
+= (({CSV.generate_line}))
+
+    # rouge ruby
+    fields = ["AAAAA"] * n_columns
+    n_rows.times do
+      CSV.generate_line(fields)
+    end
+
+  # RT
+
+  2.5, 2.6, Faster?
+
+  284.4i/s, 684.2i/s, 2.41x
+
+= (({CSV#<<}))
+
+    # rouge ruby
+
+    output = StringIO.new
+    csv = CSV.new(output)
+    n_rows.times {csv << fields}
+
+  # RT
+
+  2.5, 2.6, Faster?
+
+  2891.4i/s, 4824.1i/s, 1.67x
+
+= (({CSV.generate_line})) vs. (({CSV#<<}))
+
+  # RT
+
+  , 2.5, 2.6
+
+  (({generate_\nline})), 284.4i/s, 684.2i/s
+  (({<<})), 2891.4i/s, 4824.1i/s
+
+(('tag:center'))
+((*Use (({<<})) for multiple writes*))\n
+(('note:((*複数行書き出すときは(({<<}))を使うこと*))'))
+
+= CSV in Ruby 2.6 (3)\n(('note:Ruby 2.6のCSV (3)'))
+
+New CSV parser\n
+(('tag:small:for'))\n
+further improvements\n
+(('note:さらなる改良のための新しいCSVパーサー'))
+
+= Benchmark with KEN_ALL.CSV\n(('note:KEN_ALL.CSVでのベンチマーク'))
+
+  01101,"060  ","0600000","ﾎｯｶｲﾄﾞｳ","ｻｯﾎﾟﾛｼﾁｭｳｵｳｸ",...
+  ...(124257 lines)...
+  47382,"90718","9071801","ｵｷﾅﾜｹﾝ","ﾔｴﾔﾏｸﾞﾝﾖﾅｸﾞﾆﾁｮｳ",...
+
+(('tag:center'))
+Zip code data in Japan\n
+(('note:日本の郵便番号データ'))
+
+(('tag:center'))
+(('tag:small'))
+((<URL:https://www.post.japanpost.jp/zipcode/download.html>))
+
+= KEN_ALL.CSV statistics\n(('note:KEN_ALL.CSVの統計情報'))
+
+  # RT
+
+  Size(('note:（サイズ）')), 11.7MiB
+  (('#')) of columns(('note:（列数）')), 15
+  (('#')) of rows(('note:（行数）')), 124259
+  Encoding(('note:（エンコーディング）')), CP932
+
+= Parsing KEN_ALL.CSV\n(('note:KEN_ALL.CSVのパース'))
+
+    # rouge ruby
+    CSV.foreach("KEN_ALL.CSV",
+                "r:cp932") do |row|
+    end
+
+  # RT
+
+  2.5, 2.6, Faster?
+
+  1.17s, 0.79s, 1.48x
+
+= Fastest parsing in pure Ruby\n(('note:Ruby実装での最速のパース方法'))
+
+  # rouge ruby
+  input.each_line(chomp: true) do |line|
+    line.split(",", -1) do |row|
+    end
+  end
+
+(('tag:center'))
+Limitation: No quote\n
+(('note:制限：クォートがないこと'))
+
+= KEN_ALL.CSV without quote\n(('note:クォートなしのKEN_ALL.CSV'))
+
+  01101,060  ,0600000,ﾎｯｶｲﾄﾞｳ,ｻｯﾎﾟﾛｼﾁｭｳｵｳｸ,...
+  ...(124257 lines)...
+  47382,90718,9071801,ｵｷﾅﾜｹﾝ,ﾔｴﾔﾏｸﾞﾝﾖﾅｸﾞﾆﾁｮｳ,...
+
+= Optimized no quote CSV parsing\n(('note:最適化したクォートなしCSVのパース方法'))
+
+    # rouge ruby
+    CSV.foreach("KEN_ALL_NO_QUOTE.CSV",
+                "r:cp932",
+                quote_char: nil) {|row|}
+
+  # RT
+
+  split, 2.6, Faster?
+
+  0.32s, 0.37s, 0.86x\n(('note:(almost the same/同等)'))
+
+= Summary: Performance\n(('note:まとめ：性能'))
+
+  * Parsing: 1.5x-3x faster\n
+    (('note:パース：1.5x-3x高速'))
+    * Max to the "split" level by using an option\n
+      (('note:オプションを指定すると最大で「split」レベルまで高速化可能'))
+  * Writing: 1.5x-2.5x faster\n
+    (('note:書き出し：1.5x-2.5x高速'))
+    * Use (({CSV#<<})) than (({CSV.generate_line}))\n
+      (('note:(({CSV.generate_line}))よりも(({CSV#<<}))を使うこと'))
+
+= How to improve performance (1)
+
+Complex quote
+
+= Complex quote
+
+  "AA""AAA"
+  "AA,AAA"
+  "AA\rAAA"
+  "AA\nAAA"
+
+= Use (({StringScanner}))
+
+  * (({String#split})) is very fast
+    * But it's naive for complex quote
+
+= 2.5 uses (({String#split}))
+
+  # rouge ruby
+
+  in_extended_column = false # "...\n..." case
+  @input.each_line do |line|
+    line.split(",", -1).each do |part|
+      if in_extended_column
+        # ...
+      elsif part.start_with?('"')
+        if part.end_with?('"')
+          row << pars.gsub('""', '"') # "...""..." case
+        else
+          in_extended_column = true
+        end
+      # ...
+
+= Parsing complex quote with (({split}))
+
+  # RT
+
+  Just quoted, 274.1i/s
+  Include sep1, 211.0i/s
+  Include sep2, 118.0i/s
+
+(('tag:center'))
+Slow down
+
+= 2.6 uses (({StringScanner}))
+
+  # rouge ruby
+
+  row = []
+  until @scanner.eos?
+    value = parse_column_value
+    if @scanner.scan(/,/)
+      row << value
+    elsif @scanner.scan(/\n/)
+      row << value
+      yield(row)
+      row = []
+    end
+  end
+
+= (({parse_column_value}))
+
+  # rouge ruby
+
+  def parse_column_value
+    parse_unquoted_column_value ||
+      parse_quoted_column_value
+  end
+
+= (({parse_unquoted_column_value}))
+
+  # rouge ruby
+
+  def parse_unquoted_column_value
+    @scanner.scan(/[^,"\r\n]+/)
+  end
+
+= (({parse_unquoted_column_value}))
+
+  # rouge ruby
+
+  def parse_quoted_column_value
+    return nil unless @scanner.scan(/"/)
+    pos = @scanner.pos; value = ""
+    loop do
+      value << @scanner.scan(/[^"]+/)
+      if @scanner.scan(/""/) # "": Escaped quote
+        value << '"'
+      elsif @scanner.scan(/"/) # End
+        return value
+      else # No end quote
+        @scanner.pos = pos; return nil # Rewind and return
+      end; end; end
+
+= Parse methods can be composited
+
+  # rouge ruby
+
+  def parse_column_value
+    parse_unquoted_column_value ||
+      parse_quoted_column_value
+  end
+
+(('tag:center'))
+Easy to maintain
+
+= Parsing complex quote with (({StringScanner}))
+
+  # RT
+
+  Just quoted, 554.5i/s
+  Include sep1, 330.0i/s
+  Include sep2, 325.6i/s
+
+(('tag:center'))
+No slow down...?
+
+= How to improve performance (2)
+
+Simple case
+
+= Simple case
+
+  AAAAA
+  "AAAAA"
+
+= Use (({String#split}))
+
+(({StringScanner})) is\n
+slow for\n
+simple case
+
+= XXX
+
+  * Default gemified
+  * Reconstructed test suites for further improvements
+  * Benchmark suites for further performance improvements
+
+= Faster
+
+...
+
+= Further works
+
+  * Improve transcoding performance
+  * Implement CSV parser for simple case in C

data/theme.rb

@@ -0,0 +1 @@
+include_theme("clear-code")

metadata

@@ -0,0 +1,92 @@
+--- !ruby/object:Gem::Specification
+name: rabbit-slide-kou-rubykaigi-2019
+version: !ruby/object:Gem::Version
+  version: 2019.4.19.0
+platform: ruby
+authors:
+- Kouhei Sutou
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-04-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rabbit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.2
+- !ruby/object:Gem::Dependency
+  name: rabbit-theme-clear-code
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |-
+  csv, one of the standard libraries, in Ruby 2.6 has many improvements:
+
+    * Default gemified
+    * Faster CSV parsing
+    * Faster CSV writing
+    * Clean new CSV parser implementation for further improvements
+    * Reconstructed test suites for further improvements
+    * Benchmark suites for further performance improvements
+
+  These improvements are done without breaking backward compatibility.
+
+  This talk describes details of these improvements by a new csv maintainer.
+email:
+- kou@clear-code.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rabbit"
+- README.rd
+- Rakefile
+- config.yaml
+- csv.rab
+- images/clear-code-rubykaigi-2019-silver-sponsor.png
+- pdf/rubykaigi-2019-csv.pdf
+- theme.rb
+homepage: https://slide.rabbit-shocker.org/authors/kou/rubykaigi-2019/
+licenses:
+- CC-BY-SA-4.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6.2
+signing_key: 
+specification_version: 4
+summary: Better CSV processing with Ruby 2.6
+test_files: []