rabbit-slide-kou-scipy-japan-2020 2020.10.30.0

data/theme.rb

@@ -0,0 +1,2 @@
+@clear_code_font_family ||= find_font_family("モトヤLマルベリ3等幅")
+include_theme("clear-code")

data/tools/amazon-athena-improvement.R

@@ -0,0 +1,12 @@
+data_frame = data.frame(method=c("Athena + CSV on S3 Query",
+                                 "Federated S3 Query w/Apache Arrow"),
+                        throughput=c(0.12, 1.5))
+ggplot2::ggplot(data_frame) +
+  ggplot2::ggtitle("Apache Arrow improves data interchange performance") +
+  ggplot2::labs(x="Throughput in billion rows/sec (Longer is faster)",
+                y="Method",
+                caption="Data at https://github.com/awslabs/aws-athena-query-federation/tree/master/athena-federation-sdk#performance") +
+  ggplot2::geom_bar(ggplot2::aes(y=method, weight=throughput)) +
+  ggplot2::ggsave("images/amazon-athena-improvement.svg",
+                  dpi=100,
+                  height=3.8)

data/tools/apache-spark-improvement.R

@@ -0,0 +1,11 @@
+data_frame = data.frame(case=c("pickle", "Apache Arrow"),
+                        elapsed=c(20.7, 0.737))
+ggplot2::ggplot(data_frame) +
+  ggplot2::ggtitle("Apache Arrow improves data interchange performance") +
+  ggplot2::labs(x="Elapsed time in seconds (Shorter is faster)",
+                y="Format",
+                caption="Data at https://arrow.apache.org/blog/2017/07/26/spark-arrow/") +
+  ggplot2::geom_bar(ggplot2::aes(y=case, weight=elapsed)) +
+  ggplot2::ggsave("images/apache-spark-improvement.svg",
+                  dpi=100,
+                  height=3.8)

data/tools/benchmark-data-interchange.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+
+import datetime
+import io
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+import pyarrow.parquet as pq
+import matplotlib.pyplot as plt
+
+n_columns = 10
+n_records = 1_000_000
+# n_records = 1000
+data = np.random.randint(0, 1000, [n_records, n_columns])
+
+csv_output = io.StringIO()
+before = datetime.datetime.now()
+np.savetxt(csv_output, data, '%d', ',')
+csv_generate_elapsed_time = datetime.datetime.now() - before
+csv = csv_output.getvalue()
+print(f'Generate CSV: {csv_generate_elapsed_time.total_seconds()}')
+
+csv_input = io.StringIO(csv)
+before = datetime.datetime.now()
+np.loadtxt(csv_input, int, delimiter=',')
+csv_load_elapsed_time = datetime.datetime.now() - before
+print(f'Load CSV: {csv_load_elapsed_time.total_seconds()}')
+
+
+npy_output = io.BytesIO()
+before = datetime.datetime.now()
+np.save(npy_output, data)
+npy_generate_elapsed_time = datetime.datetime.now() - before
+npy = npy_output.getvalue()
+print(f'Generate NumPy: {npy_generate_elapsed_time.total_seconds()}')
+
+npy_input = io.BytesIO(npy)
+before = datetime.datetime.now()
+np.load(npy_input)
+npy_load_elapsed_time = datetime.datetime.now() - before
+print(f'Load NumPy: {npy_load_elapsed_time.total_seconds()}')
+
+
+before = datetime.datetime.now()
+df = pd.DataFrame(data)
+table = pa.Table.from_pandas(df)
+parquet_output = pa.BufferOutputStream()
+writer = pq.ParquetWriter(parquet_output, table.schema)
+writer.write_table(table)
+writer.close()
+parquet = parquet_output.getvalue()
+parquet_generate_elapsed_time = datetime.datetime.now() - before
+print(f'Generate Apache Parquet: {parquet_generate_elapsed_time.total_seconds()}')
+
+before = datetime.datetime.now()
+parquet_input = pa.BufferReader(parquet)
+reader = pq.ParquetFile(parquet_input)
+table = reader.read()
+df = table.to_pandas()
+df.to_numpy()
+parquet_load_elapsed_time = datetime.datetime.now() - before
+print(f'Load Apache Parquet: {parquet_load_elapsed_time.total_seconds()}')
+
+
+before = datetime.datetime.now()
+df = pd.DataFrame(data)
+table = pa.Table.from_pandas(df)
+arrow_output = pa.BufferOutputStream()
+writer = pa.ipc.new_file(arrow_output, table.schema)
+writer.write_table(table)
+writer.close()
+arrow = arrow_output.getvalue()
+arrow_generate_elapsed_time = datetime.datetime.now() - before
+print(f'Generate Apache Arrow: {arrow_generate_elapsed_time.total_seconds()}')
+
+before = datetime.datetime.now()
+arrow_input = pa.BufferReader(arrow)
+reader = pa.ipc.open_file(arrow_input)
+df = reader.read_pandas()
+df.to_numpy()
+arrow_load_elapsed_time = datetime.datetime.now() - before
+print(f'Load Apache Arrow: {arrow_load_elapsed_time.total_seconds()}')
+
+
+labels = ['Generate', 'Load']
+csv_elapsed_times = [
+    csv_generate_elapsed_time.total_seconds(),
+    csv_load_elapsed_time.total_seconds(),
+]
+npy_elapsed_times = [
+    npy_generate_elapsed_time.total_seconds(),
+    npy_load_elapsed_time.total_seconds(),
+]
+parquet_elapsed_times = [
+    parquet_generate_elapsed_time.total_seconds(),
+    parquet_load_elapsed_time.total_seconds(),
+]
+arrow_elapsed_times = [
+    arrow_generate_elapsed_time.total_seconds(),
+    arrow_load_elapsed_time.total_seconds(),
+]
+
+y = np.arange(len(labels))
+width = 0.35
+
+competities = [
+    ['csv', 'CSV', csv_elapsed_times],
+    ['numpy', 'NumPy', npy_elapsed_times],
+    ['apache-parquet', 'Apache Parquet', parquet_elapsed_times],
+]
+for id, label, elapsed_times in competities:
+    fig, ax = plt.subplots()
+    ax.barh(y - width / 2, elapsed_times, width, label=label)
+    ax.barh(y + width / 2, arrow_elapsed_times, width, label='Apache Arrow')
+
+    ax.set_xlabel(f'Elapsed time (second) ({n_records} records) (Shorter is faster)')
+    ax.set_title('Apache Arrow improves data interchange performance')
+    ax.set_yticks(y)
+    ax.set_yticklabels(labels)
+    ax.invert_yaxis()
+    ax.legend()
+
+    fig.tight_layout()
+
+    fig.savefig(f'images/benchmark-data-interchange-apache-arrow-{id}.svg')

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification
+name: rabbit-slide-kou-scipy-japan-2020
+version: !ruby/object:Gem::Version
+  version: 2020.10.30.0
+platform: ruby
+authors:
+- Sutou Kouhei
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-09-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rabbit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.2
+- !ruby/object:Gem::Dependency
+  name: rabbit-theme-clear-code
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |-
+  Apache Arrow is a cross-language development platform for in-memory
+  data. You can use Apache Arrow to process large data effectively in
+  Python and other languages such as R. Apache Arrow is the future of
+  data processing. Apache Arrow 1.0, the first major version, was
+  released at 2020-07-24. It's a good time to know Apache Arrow and
+  start using it.
+email:
+- kou@clear-code.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rabbit"
+- README.rd
+- Rakefile
+- Rplots.pdf
+- apache-arrow-1.0.rab
+- config.yaml
+- images/amazon-athena-improvement.svg
+- images/apache-arrow-and-amazon-athena.svg
+- images/apache-arrow-and-apache-parquet.svg
+- images/apache-arrow-and-apache-spark.svg
+- images/apache-arrow-and-data-interchange.svg
+- images/apache-spark-improvement.svg
+- images/arrow.svg
+- images/benchmark-data-interchange-apache-arrow-apache-parquet.svg
+- images/benchmark-data-interchange-apache-arrow-csv.svg
+- images/benchmark-data-interchange-apache-arrow-numpy.svg
+- images/contributor.png
+- images/memory-mapping.svg
+- images/noun_File_3524817.svg
+- images/noun_Memory_2294239.svg
+- images/parquet.svg
+- images/record-batch.svg
+- pdf/scipy-japan-2020-apache-arrow-1.0.pdf
+- theme.rb
+- tools/amazon-athena-improvement.R
+- tools/apache-spark-improvement.R
+- tools/benchmark-data-interchange.py
+homepage: https://slide.rabbit-shocker.org/authors/kou/scipy-japan-2020/
+licenses:
+- CC-BY-SA-4.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.0.rc.1
+signing_key: 
+specification_version: 4
+summary: Apache Arrow 1.0 - A cross-language development platform for in-memory data
+test_files: []