npy 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d8382d17139cd8c1b47882130b5612fd7d748923ead1e763a46e9f0c1e819e08
-  data.tar.gz: 26355f76370731fb6aa3eead0291cb845afa532bdc5ce7bf35a42eccc52a3819
+  metadata.gz: 2e81a0b6a617c8f05e1e62f88c05996891328ef154cd4c60d49d233b41b99748
+  data.tar.gz: 4f8c8cc599e2bae978b1cc6dabcadb2dd6aab1dec96c1f5f3e07a8a4303f1a24
 SHA512:
-  metadata.gz: 5f3e1672b1a9f378bcddd7e8783880d062dd5ffa672b72026ef5a28d862076f2200f2f86fc447feb2bea5e0113606f7bb44b8bb2c174fb1a639d634cc3e745d1
-  data.tar.gz: a4562cfb5e8511d2624e687b4fa06a304ab07075f6eb44083decc00895618b044190467d06f0793fb44c826c7b0e2f2ae5b1c61216a87c57c62f4fca05d2704c
+  metadata.gz: 82babeb75230ec73cb0870b42716589bc2bf9a460ff9943606792b9fe3dfa941f5892c4de1ba7b719afe2fa0e3726b2f5ca5ca268224afe2273c5c66af881cba
+  data.tar.gz: 700466088d563a0a8f4ccebce9506d91675b41bc1f47525756637b49218c533bfeef5364f3dea5b71328d690589643947273db00b44b7af8f86a77acdaf59ce3

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.1.1
+
+- Added support for saving files
+- Added support for npy version 2.0 and 3.0
+- Fixed error with empty shapes
+
 ## 0.1.0
 
 - First release

data/README.md

@@ -1,9 +1,11 @@
 # Npy
 
-Load NumPy `npy` and `npz` files in Ruby - no Python required
+Save and load NumPy `npy` and `npz` files in Ruby - no Python required
 
 :fire: Uses [Numo::NArray](https://github.com/ruby-numo/numo-narray) for blazing performance
 
+[![Build Status](https://travis-ci.org/ankane/npy.svg?branch=master)](https://travis-ci.org/ankane/npy)
+
 ## Installation
 
 Add this line to your application’s Gemfile:
@@ -18,6 +20,13 @@ gem 'npy'
 
 `npy` files contain a single array
 
+Save an array
+
+```ruby
+arr = Numo::Int32[0...10]
+Npy.save("x.npy", arr)
+```
+
 Load an `npy` file
 
 ```ruby
@@ -35,6 +44,14 @@ arr = Npy.load_string(byte_str)
 
 `npz` files contain multiple arrays
 
+Save multiple arrays
+
+```ruby
+x =  Numo::Int32[0...10]
+y = x * 2
+Npy.save_npz("mnist.npz", x: x, y: y)
+```
+
 Load an `npz` file
 
 ```ruby
@@ -58,6 +75,8 @@ Arrays are lazy loaded for performance
 ## Resources
 
 - [npy format](https://docs.scipy.org/doc/numpy/reference/generated/numpy.lib.format.html#module-numpy.lib.format)
+- [NumPy data types](https://docs.scipy.org/doc/numpy/user/basics.types.html)
+- [Numo::NArray docs](https://ruby-numo.github.io/narray/narray/Numo/NArray.html)
 
 ## History
 

data/lib/npy.rb

@@ -11,6 +11,21 @@ module Npy
 
   MAGIC_STR = "\x93NUMPY".b
 
+  TYPE_MAP = {
+    "|i1" => Numo::Int8,
+    "<i2" => Numo::Int16,
+    "<i4" => Numo::Int32,
+    "<i8" => Numo::Int64,
+    "|u1" => Numo::UInt8,
+    "<u2" => Numo::UInt16,
+    "<u4" => Numo::UInt32,
+    "<u8" => Numo::UInt64,
+    "<f4" => Numo::SFloat,
+    "<f8" => Numo::DFloat,
+    "<c8" => Numo::SComplex,
+    "<c16" => Numo::DComplex
+  }
+
   class << self
     def load(path)
       with_file(path) do |f|
@@ -37,54 +52,90 @@ module Npy
       magic = io.read(6)
       raise Error, "Invalid npy format" unless magic&.b == MAGIC_STR
 
-      major_version = io.read(1)
-      minor_version = io.read(1)
-      raise Error, "Unsupported version" unless major_version == "\x01".b
+      version = io.read(2)
 
-      header_len = io.read(2).unpack1("S<")
+      header_len =
+        case version
+        when "\x01\x00".b
+          io.read(2).unpack1("S<")
+        when "\x02\x00".b, "\x03\x00".b
+          io.read(4).unpack1("I<")
+        else
+          raise Error, "Unsupported version"
+        end
       header = io.read(header_len)
       descr, fortran_order, shape = parse_header(header)
       raise Error, "Fortran order not supported" if fortran_order
 
-      klass =
-        case descr
-        when "|i1"
-          Numo::Int8
-        when "<i2"
-          Numo::Int16
-        when "<i4"
-          Numo::Int32
-        when "<i8"
-          Numo::Int64
-        when "|u1"
-          Numo::UInt8
-        when "<u2"
-          Numo::UInt16
-        when "<u4"
-          Numo::UInt32
-        when "<u8"
-          Numo::UInt64
-        when "<f4"
-          Numo::SFloat
-        when "<f8"
-          Numo::DFloat
-        when "<c8"
-          Numo::SComplex
-        when "<c16"
-          Numo::DComplex
-        else
-          raise Error, "Type not supported: #{descr}"
-        end
+      # numo can't handle empty shapes
+      empty_shape = shape.empty?
+      shape = [1] if empty_shape
 
-      klass.from_binary(io.read, shape)
+      klass = TYPE_MAP[descr]
+      raise Error, "Type not supported: #{descr}" unless klass
+
+      # use from_string instead of from_binary for max compatibility
+      # from_binary introduced in 0.9.0.4
+      result = klass.from_string(io.read, shape)
+      result = result[0] if empty_shape
+      result
     end
 
     def load_npz_io(io)
       File.new(io)
     end
 
+    def save(path, arr)
+      ::File.open(path, "wb") do |f|
+        save_io(f, arr)
+      end
+      true
+    end
+
+    def save_npz(path, **arrs)
+      # use File.open instead passing path to zip file
+      # so it overrides instead of appends
+      ::File.open(path, "wb") do |f|
+        Zip::File.open(f, Zip::File::CREATE) do |zipfile|
+          arrs.each do |k, v|
+            zipfile.get_output_stream("#{k}.npy") do |f2|
+              save_io(f2, v)
+            end
+          end
+        end
+      end
+      true
+    end
+
     private
 
+    def save_io(f, arr)
+      empty_shape = arr.is_a?(Numeric)
+      arr = Numo::NArray.cast([arr]) if empty_shape
+      arr = Numo::NArray.cast(arr) if arr.is_a?(Array)
+
+      # desc
+      descr = TYPE_MAP.find { |k, v| arr.is_a?(v) }
+      raise Error, "Unsupported type: #{arr.class.name}" unless descr
+
+      # shape
+      shape = arr.shape
+      shape << "" if shape.size == 1
+      shape = [] if empty_shape
+
+      # header
+      header = "{'descr': '#{descr[0]}', 'fortran_order': False, 'shape': (#{shape.join(", ")}), }".b
+      padding_len = 64 - (11 + header.length) % 64
+      padding = "\x20".b * padding_len
+      header = "#{header}#{padding}\n"
+
+      f.write(MAGIC_STR)
+      f.write("\x01\x00".b)
+      f.write([header.bytesize].pack("S<"))
+      f.write(header)
+      f.write(arr.to_string)
+    end
+
     def with_file(path)
       ::File.open(path, "rb") do |f|
         yield f
@@ -93,17 +144,21 @@ module Npy
 
     # header is Python dict, so use regex to parse
     def parse_header(header)
+      # sanity check
+      raise Error, "Bad header" if !header || header[-1] != "\n"
+
       # descr
-      m = /'descr': '([^']+)'/.match(header)
+      m = /'descr': *'([^']+)'/.match(header)
       descr = m[1]
 
       # fortran_order
-      m = /'fortran_order': ([^,]+)/.match(header)
+      m = /'fortran_order': *([^,]+)/.match(header)
       fortran_order = m[1] == "True"
 
       # shape
-      m = /'shape': \(([^)]+)\)/.match(header)
-      shape = m[1].split(", ").map(&:to_i)
+      m = /'shape': *\(([^)]*)\)/.match(header)
+      # no space in split for max compatibility
+      shape = m[1].split(",").map(&:to_i)
 
       [descr, fortran_order, shape]
     end

data/lib/npy/version.rb

@@ -1,3 +1,3 @@
 module Npy
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: npy
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-13 00:00:00.000000000 Z
+date: 2019-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray
@@ -114,5 +114,5 @@ requirements: []
 rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
-summary: Load NumPy npy and npz files in Ruby
+summary: Save and load NumPy npy and npz files in Ruby
 test_files: []