fast_excel 0.1.7 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 56b8736c80626c089a7b20544519504306eaf41f
-  data.tar.gz: 3acc3eceba37ae987a269a75b846713624d6d3fc
+  metadata.gz: 6c5919483e8d5ac1277efefe8399926f14b45d10
+  data.tar.gz: 2a8bd5f6067c8903778f3f7227f90d4c95d31868
 SHA512:
-  metadata.gz: 9d4d65513075a6935d344f6d40193db2a4095c3ea66ecb36e5eff2308660704741257b46b83b6535e83c571d945e1f1ada48c95db9aeea48c2eb0299a89b3fc4
-  data.tar.gz: 67792c48ae5170cffb0b8f51701f6e0f44489ed7f7d6b4a55ca814f32e987b271053755939d9fde4e4230ad7ab554827d5f9d9df962df9eb74a1ae47d194602b
+  metadata.gz: dd0f959fe4eb08bd7506f63b4f666c3d47c05765aa8213c775f873264dd561e8aa2c9dd7f61d1ea5f1e6498199248dfef80bb9b5f7693025ec0c3f192671dd27
+  data.tar.gz: 5168323f914545536e1f36ed30aa0f689a98e92a93875abb7137109c20c2adfbd78b905a6cf6339c36e9ae391c23d7461e24a6d15ad92cc28e43ef955b16ce1c

data/Gemfile

@@ -1,9 +1,15 @@
 source 'https://rubygems.org'
 
 gem 'ffi'
-gem 'ffi_gen'
-gem 'axlsx'
+gem 'ffi_gen', require: false
+
+gem 'roo', '2.7.1', git: 'https://github.com/roo-rb/roo.git'
 
-gem 'roo'
 gem 'minitest'
-gem 'minitest-reporters'
+gem 'minitest-reporters'
+
+# For benchmakrs
+gem 'axlsx', git: 'https://github.com/randym/axlsx.git'
+gem 'write_xlsx'
+gem 'benchmark-ips'
+gem 'process_memory', git: 'https://github.com/paxa/process_memory'

data/Gemfile.lock

@@ -1,16 +1,38 @@
+GIT
+  remote: https://github.com/paxa/process_memory
+  revision: 1aa4df28c9a98903301317236104abe591e086fd
+  specs:
+    process_memory (0.1)
+
+GIT
+  remote: https://github.com/randym/axlsx.git
+  revision: c8ac844572b25fda358cc01d2104720c4c42f450
+  specs:
+    axlsx (2.1.0.pre)
+      htmlentities (~> 4.3.4)
+      mimemagic (~> 0.3)
+      nokogiri (>= 1.6.6)
+      rubyzip (>= 1.2.1)
+
+GIT
+  remote: https://github.com/roo-rb/roo.git
+  revision: 6bde8588dfd97d7a18981ee3ac7b444fec165a2c
+  specs:
+    roo (2.7.1)
+      nokogiri (~> 1)
+      rubyzip (>= 1.2.1, < 2.0.0)
+
 GEM
   remote: https://rubygems.org/
   specs:
     ansi (1.5.0)
-    axlsx (2.0.1)
-      htmlentities (~> 4.3.1)
-      nokogiri (>= 1.4.1)
-      rubyzip (~> 1.0.0)
+    benchmark-ips (2.7.2)
     builder (3.2.3)
-    ffi (1.9.17)
+    ffi (1.9.18)
     ffi_gen (1.2.0)
       ffi (~> 1.0)
     htmlentities (4.3.4)
+    mimemagic (0.3.2)
     mini_portile2 (2.1.0)
     minitest (5.10.1)
     minitest-reporters (1.1.14)
@@ -18,28 +40,29 @@ GEM
       builder
       minitest (>= 5.0)
       ruby-progressbar
-    nokogiri (1.7.0.1)
+    nokogiri (1.7.1)
       mini_portile2 (~> 2.1.0)
-    roo (1.13.2)
-      nokogiri
-      rubyzip
-      spreadsheet (> 0.6.4)
-    ruby-ole (1.2.12.1)
     ruby-progressbar (1.8.1)
-    rubyzip (1.0.0)
-    spreadsheet (1.1.4)
-      ruby-ole (>= 1.0)
+    rubyzip (1.2.1)
+    write_xlsx (0.83.0)
+      rubyzip (>= 1.0.0)
+      zip-zip
+    zip-zip (0.3)
+      rubyzip (>= 1.0.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  axlsx
+  axlsx!
+  benchmark-ips
   ffi
   ffi_gen
   minitest
   minitest-reporters
-  roo
+  process_memory!
+  roo (= 2.7.1)!
+  write_xlsx
 
 BUNDLED WITH
    1.14.5

data/Makefile

@@ -8,7 +8,7 @@ all :
 	$(Q)$(MAKE) -C libxlsxwriter
 
 clean :
-	#$(Q)$(MAKE) clean -C libxlsxwriter
+	$(Q)$(MAKE) clean -C libxlsxwriter
 
 install :
 	@echo "Nothing to install"

data/README.md

@@ -22,18 +22,6 @@ worksheet.write_row(0, ["message", "price", "date"], bold)
 
 for i in 1..1000
   worksheet.write_row(i, ["Hello", (rand * 10_000_000).round(2), Time.now])
-
-  # Or manually
-  # worksheet.write_string(i, 0, "Hello", nil)
-  # worksheet.write_number(i, 1, (rand * 10_000_000).round(2), nil)
-  # date = Libxlsxwriter::Datetime.new
-  # date[:year] = 2017
-  # date[:month] = 2
-  # date[:day] = 24
-  # date[:hour] = i % 24
-  # date[:min] = i % 60
-  # date[:sec] = i % 60
-  # worksheet.write_datetime(i, 2, date, nil)
 end
 
 workbook.close
@@ -41,3 +29,63 @@ workbook.close
 
 This repositiry and gem contain sources of [libxlsxwriter](https://github.com/jmcnamara/libxlsxwriter)
 
+## Benchmarks
+
+1000 rows:
+```
+Comparison:
+           FastExcel:       31.7 i/s
+               Axslx:        8.0 i/s - 3.98x  slower
+          write_xlsx:        6.9 i/s - 4.62x  slower
+```
+
+20000 rows:
+```
+Comparison:
+           FastExcel:        1.4 i/s
+               Axslx:        0.4 i/s - 3.46x  slower
+          write_xlsx:        0.1 i/s - 17.04x  slower
+```
+
+Max memory usage, generating 100k rows:
+```
+FastExcel   - 20 MB
+Axslx       - 60 MB
+write_axslx - 100 MB
+```
+
+## Install
+
+```ruby
+# Gemfile
+gem 'fast_excel'
+```
+Or
+```
+gem install fast_excel
+```
+
+## API
+
+This gem is FFI binding for libxlsxwriter C library with some syntax sugar. All original functions is avaliable, for example:
+
+```ruby
+Libxlsxwriter.worksheet_activate(worksheet) # => will call void worksheet_activate(lxw_worksheet *worksheet)
+# or shorter:
+worksheet.activate
+```
+
+Full libxlsxwriter documentation: [http://libxlsxwriter.github.io/](http://libxlsxwriter.github.io/)
+
+Helper Methods:
+
+* `FastExcel.open(filename = nil, constant_memory: false, default_format: {})` - open new workbook, if `filename` is nil - it will craete tmp file, `default_format` will be called with `workbook.default_format.set(...)`
+* `FastExcel.date_num(time, offset = nil)` - generate Excel's internal date value, number of days since 1900-Jan-01, works faster then creating `Libxlsxwriter::Datetime` struct. `offset` argument is number hours from UTC, e.g. `3.5`
+* `FastExcel.print_ffi_obj(object)` - print FFI object fields, just for debugging
+* `workbook.bold_cell_format` - shortcut for creating bold format
+* `workbook.number_format(num_format)` - create number or date format, for money usually: `"#,##0.00"`, for date: `"[$-409]m/d/yy h:mm AM/PM;@"`
+* `workbook.read_string` - close workbook, read file to string, delete file (only if tmp file)
+* `workbook.remove_tmp_file` - delete tmp file (only if tmp file)
+* `worksheet.write_row(row_num, array_of_mixed_value, formats = nil)` - write values one by one, detecting type automatically. `formats` can be array, or Format object or nil
+* `format.font_family` - alias for `format.font_name`
+* `workbook.default_format.font_size` - set it to 0 if you want to use default font size (that what user set in Excel settings)

data/Rakefile

@@ -1,31 +1,4 @@
-namespace :build do
-  desc "Build mac binary"
-  task :mac do
-    Dir.chdir("./libxlsxwriter") do
-      system("make clean && make")
-      system('cp lib/libxlsxwriter.dylib ../binaries/libxlsxwriter-darwin.dylib')
-    end
-  end
-
-  desc "Build linux (libc) binary"
-  task :linux do
-    system("docker build . -f build_centos_linux.docker -t fast_excel_centos")
-    system("docker run -t fast_excel_centos readelf -d lib/libxlsxwriter.so")
-    last_container_id = `docker ps -a | grep fast_excel_centos | head -1 | awk '{print $1;}'`.strip
-    system("docker cp #{last_container_id}:/srv/libxlsxwriter/lib/libxlsxwriter.so ./binaries/libxlsxwriter-glibc.so")
-    system("docker rm #{last_container_id}")
-  end
-
-  desc "Build linux (musl) binary"
-  task :linux_musl do
-    system("docker build . -f build_alpine_linux.docker -t fast_excel_alpine")
-    system("docker run -t fast_excel_alpine readelf -d lib/libxlsxwriter.so")
-    last_container_id = `docker ps -a | grep fast_excel_alpine | head -1 | awk '{print $1;}'`.strip
-    system("docker cp #{last_container_id}:/srv/libxlsxwriter/lib/libxlsxwriter.so ./binaries/libxlsxwriter-alpine.so")
-    system("docker rm #{last_container_id}")
-  end
-end
-
+desc "Sync github.com:Paxa/libxlsxwriter to ./libxlsxwriter"
 task :sync do
   require 'fileutils'
   FileUtils.rm_rf("./libxlsxwriter")
@@ -34,4 +7,12 @@ task :sync do
     system("git show --pretty='format:%cd %h' --date=iso --quiet > version.txt")
     FileUtils.rm_rf("./.git")
   end
-end
+end
+
+require 'rake/testtask'
+
+Rake::TestTask.new do |test|
+  test.test_files = Dir.glob('test/**/*_test.rb')
+end
+
+#task :default => :test

data/benchmarks/1k_rows.rb

@@ -0,0 +1,59 @@
+require_relative 'init'
+
+HEADERS = ["id", "name", "age", "date"]
+
+DATA = []
+1000.times do |n|
+  DATA << [n, "String string #{n}", (n * rand * 10).round, Time.at(n * 1000 + 1492922688)]
+end
+
+Benchmark.ips do |x|
+  x.config(time: 10, warmup: 2)
+
+  x.report("FastExcel") do
+    workbook = FastExcel.open(constant_memory: true)
+    worksheet = workbook.add_worksheet("benchmark")
+
+    worksheet.write_row(0, HEADERS)
+    DATA.each_with_index do |row, i|
+      worksheet.write_row(i + 1, row)
+    end
+    workbook.read_string
+  end
+
+  x.report("Axslx") do
+    filename = "#{Dir.mktmpdir}/axslx.xslx"
+    Axlsx::Package.new do |package|
+      package.use_autowidth = false
+      package.workbook.add_worksheet do |sheet|
+        sheet.add_row(HEADERS)
+        DATA.each do |row|
+          sheet.add_row(row)
+        end
+      end
+      package.serialize(filename)
+      File.open(filename, 'rb', &:read)
+      File.delete(filename)
+    end
+  end
+
+  x.report("write_xlsx") do
+    filename = "#{Dir.mktmpdir}/write_xlsx.xslx"
+    workbook = WriteXLSX.new(filename)
+    worksheet = workbook.add_worksheet
+    HEADERS.each_with_index do |value, i|
+      worksheet.write(0, i, value)
+    end
+    DATA.each_with_index do |row, row_num|
+      worksheet.write_number(row_num + 1, 0, row[0])
+      worksheet.write_string(row_num + 1, 1, row[1])
+      worksheet.write_number(row_num + 1, 2, row[2])
+      worksheet.write_number(row_num + 1, 3, row[3])
+    end
+    workbook.close
+    File.open(filename, 'rb', &:read)
+    File.delete(filename)
+  end
+
+  x.compare!
+end

data/benchmarks/20k_rows.rb

@@ -0,0 +1,26 @@
+require_relative 'init'
+
+HEADERS = ["id", "name", "age", "date", "random"]
+
+DATA = []
+20_000.times do |n|
+  DATA << [n, "String string #{n}" * 5, (n * rand * 10).round, Time.at(n * 1000 + 1492922688), n * 100]
+end
+
+Benchmark.ips do |x|
+  x.config(time: 10, warmup: 2)
+
+  x.report("FastExcel") do
+    write_fast_excel_20k
+  end
+
+  x.report("Axslx") do
+    write_axslx_20k
+  end
+
+  x.report("write_xlsx") do
+    write_xlsx_20k
+  end
+
+  x.compare!
+end

data/benchmarks/init.rb

@@ -0,0 +1,59 @@
+require 'bundler/setup'
+require 'fileutils'
+require 'tmpdir'
+require_relative '../lib/fast_excel'
+
+# gem install axlsx benchmark-ips write_xlsx
+
+require "benchmark/ips"
+require 'axlsx'
+require 'write_xlsx'
+require 'process_memory'
+
+require_relative 'init'
+
+def write_fast_excel_20k
+  workbook = FastExcel.open(constant_memory: true)
+  worksheet = workbook.add_worksheet("benchmark")
+
+  worksheet.write_row(0, HEADERS)
+  DATA.each_with_index do |row, i|
+    worksheet.write_row(i + 1, row)
+  end
+  workbook.read_string
+end
+
+def write_xlsx_20k
+  filename = "#{Dir.mktmpdir}/write_xlsx.xslx"
+  workbook = WriteXLSX.new(filename)
+  worksheet = workbook.add_worksheet
+  HEADERS.each_with_index do |value, i|
+    worksheet.write(0, i, value)
+  end
+  DATA.each_with_index do |row, row_num|
+    worksheet.write_number(row_num + 1, 0, row[0])
+    worksheet.write_string(row_num + 1, 1, row[1])
+    worksheet.write_number(row_num + 1, 2, row[2])
+    worksheet.write_number(row_num + 1, 3, row[3])
+    worksheet.write_number(row_num + 1, 4, row[4])
+  end
+  workbook.close
+  File.open(filename, 'rb', &:read)
+  File.delete(filename)
+end
+
+def write_axslx_20k
+  filename = "#{Dir.mktmpdir}/axslx.xslx"
+  Axlsx::Package.new do |package|
+    package.use_autowidth = false
+    package.workbook.add_worksheet do |sheet|
+      sheet.add_row(HEADERS)
+      DATA.each do |row|
+        sheet.add_row(row)
+      end
+    end
+    package.serialize(filename)
+    File.open(filename, 'rb', &:read)
+    File.delete(filename)
+  end
+end

data/benchmarks/memory.rb

@@ -0,0 +1,49 @@
+require_relative 'init'
+
+HEADERS = ["id", "name", "age", "date", "random"]
+
+DATA = []
+10_000.times do |n|
+  DATA << [n, "String string #{n}" * 5, (n * rand * 10).round, Time.at(n * 1000 + 1492922688), n * 100]
+end
+
+puts "warm up..."
+write_fast_excel_20k
+write_axslx_20k
+write_xlsx_20k
+
+DATA.clear
+100_000.times do |n|
+  DATA << [n, "String string #{n}" * 5, (n * rand * 10).round, Time.at(n * 1000 + 1492922688), n * 100]
+end
+
+GC.start
+sleep 5
+
+def measure_memory(title)
+  puts "Running test: #{title}"
+  recorder = ProcessMemory.start_recording
+  yield
+  puts recorder.print("Done!")
+  recorder.stop
+  puts recorder.report_per_second_pretty
+  puts
+end
+
+measure_memory("FastExcel") do
+  write_fast_excel_20k
+end
+
+GC.start
+sleep 5
+
+measure_memory("Axslx") do
+  write_axslx_20k
+end
+
+GC.start
+sleep 5
+
+measure_memory("write_axslx") do
+  write_axslx_20k
+end