zfp 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +54 -0
- data/LICENSE +21 -0
- data/README.md +71 -43
- data/lib/zfp/codec.rb +18 -14
- data/lib/zfp/errors.rb +1 -0
- data/lib/zfp/field.rb +25 -13
- data/lib/zfp/packed.rb +72 -0
- data/lib/zfp/packer.rb +38 -11
- data/lib/zfp/stream.rb +49 -7
- data/lib/zfp/type_coercion.rb +47 -14
- data/lib/zfp/version.rb +1 -1
- data/lib/zfp.rb +14 -3
- metadata +103 -84
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e9728265d45dba0b43c273189c1f546d7c01a76d31d06beabab266a947489521
|
|
4
|
+
data.tar.gz: 2e98e94b05746379756e9d15584922d51c2ad956c76f48e2d067fa31061aa1bf
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 44f4a539fbddf1a43ae612d560ed34da485d0a9ba7dfbfa73dbc1e50db843ef0da4301a9d716f2979b23f67248af88511309a39f3721d1c80b86a887fc5c5f02
|
|
7
|
+
data.tar.gz: eb4c1b72943bd93a9f71c5eb12f4aeba1aa011f7430b9543da80b8dcde1d0c64f7aee07850b8db53c4ebd945e872618741db80d4f233f95a64093604cf2783cf
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
## [Unreleased]
|
|
2
|
+
|
|
3
|
+
## [0.2.0] - 2026-10-31
|
|
4
|
+
|
|
5
|
+
### Fixed
|
|
6
|
+
|
|
7
|
+
- **Multi-dimensional data was handed to libzfp transposed** ([#2](https://github.com/MadBomber/zfp/issues/2)).
|
|
8
|
+
`shape:` is row-major (Ruby Array / Numo order, last dimension fastest) but
|
|
9
|
+
`shape[0]` was passed as zfp's `nx`, which is the fastest-varying dimension.
|
|
10
|
+
Round-trips were still exact, but compression ratios and lossy accuracy on
|
|
11
|
+
2-D/3-D/4-D data were badly degraded. The gem now reverses the shape
|
|
12
|
+
internally; the public `shape:` contract is unchanged.
|
|
13
|
+
- **Shape is now validated against the data length.** A mismatch used to make
|
|
14
|
+
libzfp read or write past the data buffer (silent garbage, or a bus error for
|
|
15
|
+
large shapes). It now raises `Zfp::InvalidShape`.
|
|
16
|
+
- **Decompression no longer reads past the input.** libzfp's bitstream has no
|
|
17
|
+
bounds checking; short or corrupt input is now copied into a zero-filled
|
|
18
|
+
buffer of the worst-case stream size. Empty input raises
|
|
19
|
+
`Zfp::DecompressionFailed`, and so does a truncated `:fixed_rate` stream,
|
|
20
|
+
whose exact size is known.
|
|
21
|
+
- Integer types (`:int32`, `:int64`) now raise `Zfp::InvalidMode` for any mode
|
|
22
|
+
other than `:reversible`, as documented. Previously lossy modes were silently
|
|
23
|
+
applied.
|
|
24
|
+
- Integer data is validated: non-Integer elements or values outside the 32- or
|
|
25
|
+
64-bit range raise `Zfp::InvalidType` instead of being truncated or wrapped.
|
|
26
|
+
- `numo: true` without numo-narray loaded raises `Zfp::NumoNotAvailable`
|
|
27
|
+
instead of a bare `NameError`; the gem now tries to `require "numo/narray"`
|
|
28
|
+
on demand.
|
|
29
|
+
- `Zfp.pack` accepts an explicit `numo:` keyword.
|
|
30
|
+
|
|
31
|
+
### Changed
|
|
32
|
+
|
|
33
|
+
- **Pack header magic bumped to `ZFP\x02`.** Multi-dimensional bytes packed by
|
|
34
|
+
0.1.0 (`ZFP\x01`) were compressed with the transposed layout and are rejected
|
|
35
|
+
with a clear `Zfp::PackerError`; 1-D 0.1.0 bytes are layout-identical and
|
|
36
|
+
still unpack.
|
|
37
|
+
- Shape validation lives in `Zfp::Field.validate_shape!` and is shared by
|
|
38
|
+
`Zfp::Codec`.
|
|
39
|
+
|
|
40
|
+
### Added
|
|
41
|
+
|
|
42
|
+
- **Save and load `.zfp` files** ([#1](https://github.com/MadBomber/zfp/issues/1)).
|
|
43
|
+
`Zfp.pack` and `Codec#pack` now return a `Zfp::Packed` instead of a raw
|
|
44
|
+
`String`. It converts to a `String` via `#to_str`, so Redis, `File.binwrite`,
|
|
45
|
+
`Zfp.unpack` and friends keep working; code that indexed the result as a
|
|
46
|
+
String (`packed[0, 4]`) needs `packed.to_str`. `Packed#save(path)` writes the
|
|
47
|
+
file and `Zfp.load(path)` (alias `Packed.load`) reads it back with metadata
|
|
48
|
+
intact. `Packed#type`, `#shape`, `#mode`, `#params`, `#numo?` and `#inspect`
|
|
49
|
+
read the header without decompressing; `Packer.decode_header` backs them.
|
|
50
|
+
- `LICENSE` file (MIT) and gemspec license/metadata.
|
|
51
|
+
|
|
52
|
+
## [0.1.0] - 2026-04-28
|
|
53
|
+
|
|
54
|
+
- Initial release
|
data/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Dewayne VanHoozer
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
|
|
1
|
+
<div align="center">
|
|
2
|
+
<img src="docs/assets/images/zfp.png" alt="zfp logo" width="1200"/>
|
|
3
|
+
<br/>
|
|
4
|
+
<i>Because your floats deserve better than Base64.</i>
|
|
5
|
+
</div>
|
|
6
|
+
<p>
|
|
2
7
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
[](https://rubygems.org/gems/zfp)
|
|
6
|
-
[](LICENSE)
|
|
7
|
-
|
|
8
|
-
**`zfp`** brings [LLNL's battle-hardened ZFP compression library](https://computing.llnl.gov/projects/zfp) to Ruby. ZFP was built by national-lab scientists to compress petabytes of floating-point simulation data without losing the ability to do science on it. Now it's in a Ruby gem. You're welcome, science.
|
|
8
|
+
The ZFP gem brings [LLNL's battle-hardened ZFP compression library](https://computing.llnl.gov/projects/zfp) to Ruby. ZFP was built by national-lab scientists to compress petabytes of floating-point simulation data without losing the ability to do science on it. Now it's in a Ruby gem. You're welcome, science.
|
|
9
9
|
|
|
10
10
|
Whether you're cramming ten years of OHLCV market data into Redis, shipping a million embedding vectors over the wire, or just deeply offended by how wasteful `Array#pack("E*")` is, this gem is for you.
|
|
11
11
|
|
|
@@ -28,10 +28,11 @@ It offers four compression modes:
|
|
|
28
28
|
|
|
29
29
|
## Installation
|
|
30
30
|
|
|
31
|
-
You need `libzfp` installed first
|
|
31
|
+
You need `libzfp` installed first, with [upkg](https://github.com/seuros/upkg) or Homebrew:
|
|
32
32
|
|
|
33
33
|
```bash
|
|
34
|
-
upkg install zfp
|
|
34
|
+
upkg install zfp # any platform
|
|
35
|
+
brew install zfp # macOS
|
|
35
36
|
```
|
|
36
37
|
|
|
37
38
|
Then add the gem:
|
|
@@ -65,9 +66,13 @@ prices == restored # => true
|
|
|
65
66
|
|
|
66
67
|
# Self-describing pack — no metadata bookkeeping required
|
|
67
68
|
packed = Zfp.pack(prices, type: :double, shape: [prices.size], mode: :reversible)
|
|
68
|
-
restored =
|
|
69
|
+
restored = packed.unpack # type, shape, mode all embedded — no args needed
|
|
69
70
|
|
|
70
71
|
prices == restored # => true
|
|
72
|
+
|
|
73
|
+
# Save to disk and load it back — a .zfp file needs no sidecar schema
|
|
74
|
+
packed.save("prices.zfp")
|
|
75
|
+
Zfp.load("prices.zfp").unpack == prices # => true
|
|
71
76
|
```
|
|
72
77
|
|
|
73
78
|
That's it. The rest is just choosing how hard you want to squeeze.
|
|
@@ -93,18 +98,28 @@ bytes = Zfp.compress(prices, type: :double, shape: [1000], mode: :fixed_accuracy
|
|
|
93
98
|
data = Zfp.decompress(bytes, type: :double, shape: [1000], mode: :fixed_accuracy, tolerance: 0.001)
|
|
94
99
|
```
|
|
95
100
|
|
|
96
|
-
**Self-describing bytes** (`pack` / `unpack`)
|
|
101
|
+
**Self-describing bytes** (`pack` / `unpack` / `save` / `load`)
|
|
97
102
|
|
|
98
|
-
|
|
103
|
+
`pack` returns a `Zfp::Packed`: the compressed payload behind a 32-byte header holding type, shape, mode, and parameters. Pass it anywhere a `String` goes — Redis, S3, a message queue, `File.binwrite` — it converts via `to_str`. Unpack without needing a schema.
|
|
99
104
|
|
|
100
105
|
```ruby
|
|
101
106
|
# Pack: type/shape/mode/params stored in the bytes themselves
|
|
102
|
-
packed
|
|
107
|
+
packed = Zfp.pack(data, type: :double, shape: [1000], mode: :fixed_accuracy, tolerance: 0.001)
|
|
108
|
+
packed # => #<Zfp::Packed type=:double shape=[1000] mode=:fixed_accuracy tolerance=0.001 bytesize=2744>
|
|
109
|
+
packed.shape # header fields are readable without decompressing
|
|
110
|
+
packed.bytesize
|
|
103
111
|
|
|
104
112
|
# Unpack: zero arguments needed
|
|
105
|
-
restored = Zfp.unpack(packed)
|
|
113
|
+
restored = packed.unpack # or Zfp.unpack(packed), or Zfp.unpack(raw_string)
|
|
114
|
+
|
|
115
|
+
# Persist as a .zfp file and read it back
|
|
116
|
+
packed.save("prices.zfp")
|
|
117
|
+
loaded = Zfp.load("prices.zfp") # => Zfp::Packed, metadata intact
|
|
118
|
+
loaded.unpack
|
|
106
119
|
```
|
|
107
120
|
|
|
121
|
+
The header is specific to this gem. Neither it nor the raw `compress` output (which carries no zfp header at all) can be read by the `zfp` command-line tool or by zfp bindings for other languages.
|
|
122
|
+
|
|
108
123
|
### `Zfp::Codec` — when you compress many arrays with the same config
|
|
109
124
|
|
|
110
125
|
Build a codec once. Compress forever.
|
|
@@ -120,8 +135,8 @@ end
|
|
|
120
135
|
# Retrieve and decompress
|
|
121
136
|
prices = codec.decompress(store["AAPL"])
|
|
122
137
|
|
|
123
|
-
# Or pack (self-describing) via codec
|
|
124
|
-
|
|
138
|
+
# Or pack (self-describing) via codec, and save it
|
|
139
|
+
codec.pack(daily_closes["AAPL"]).save("aapl.zfp")
|
|
125
140
|
```
|
|
126
141
|
|
|
127
142
|
---
|
|
@@ -145,6 +160,8 @@ Zfp.compress(tensor, type: :double, shape: [10, 30, 252], mode: :reversible)
|
|
|
145
160
|
Zfp.compress(data4d, type: :double, shape: [4, 8, 16, 16], mode: :reversible)
|
|
146
161
|
```
|
|
147
162
|
|
|
163
|
+
`shape:` is **row-major**: the order of a nested Ruby Array after `flatten`, and the order `Numo::NArray#shape` reports, with the last dimension varying fastest. libzfp numbers its axes the other way round (its `nx` is the fastest-varying one); the gem does that conversion internally, so never reverse a shape yourself. The product of the dimensions must equal the element count, or `Zfp::InvalidShape` is raised. See [Multi-Dimensional Arrays](https://madbomber.github.io/zfp/multi-dimensional/) for why this matters: described correctly, a smooth 100×120 grid compresses 15×; transposed, 6×.
|
|
164
|
+
|
|
148
165
|
---
|
|
149
166
|
|
|
150
167
|
## All Four Scalar Types
|
|
@@ -154,7 +171,8 @@ Zfp.compress(data4d, type: :double, shape: [4, 8, 16, 16], mode: :reversible)
|
|
|
154
171
|
Zfp.compress(floats, type: :float, shape: [n], mode: :reversible)
|
|
155
172
|
Zfp.compress(doubles, type: :double, shape: [n], mode: :fixed_accuracy, tolerance: 1e-6)
|
|
156
173
|
|
|
157
|
-
# Integer — reversible mode only
|
|
174
|
+
# Integer — reversible mode only; any other mode raises Zfp::InvalidMode.
|
|
175
|
+
# Elements must be Integers within the 32-/64-bit signed range, or Zfp::InvalidType is raised.
|
|
158
176
|
Zfp.compress(counts, type: :int32, shape: [n], mode: :reversible)
|
|
159
177
|
Zfp.compress(ids, type: :int64, shape: [n], mode: :reversible)
|
|
160
178
|
```
|
|
@@ -163,7 +181,7 @@ Zfp.compress(ids, type: :int64, shape: [n], mode: :reversible)
|
|
|
163
181
|
|
|
164
182
|
## Numo::NArray Support
|
|
165
183
|
|
|
166
|
-
If [numo-narray](https://github.com/ruby-numo/numo-narray)
|
|
184
|
+
If you have required [numo-narray](https://github.com/ruby-numo/numo-narray), the gem auto-detects type and shape from Numo arrays. No `type:` or `shape:` arguments needed on compress or pack. For Numo *output* (`numo: true`, or unpacking bytes that were packed from a Numo array) the gem requires `numo/narray` on demand and raises `Zfp::NumoNotAvailable` if the gem is not installed.
|
|
167
185
|
|
|
168
186
|
```ruby
|
|
169
187
|
require "numo/narray"
|
|
@@ -195,12 +213,14 @@ result = Zfp.unpack(packed)
|
|
|
195
213
|
|
|
196
214
|
### `:reversible` — lossless, bit-exact
|
|
197
215
|
|
|
198
|
-
Use this when correctness is non-negotiable. Compression ratios depend on
|
|
199
|
-
|
|
216
|
+
Use this when correctness is non-negotiable. Compression ratios depend entirely on how
|
|
217
|
+
predictable neighboring values are: smooth grids and sequences compress well; noisy
|
|
218
|
+
series and high-entropy data barely shrink.
|
|
200
219
|
|
|
201
220
|
```ruby
|
|
202
221
|
bytes = Zfp.compress(data, type: :double, shape: [n], mode: :reversible)
|
|
203
|
-
#
|
|
222
|
+
# Measured: 1.1x on simulated daily closes, 2.7x–5.8x on arithmetic sequences,
|
|
223
|
+
# 15x on a smooth 2-D grid. Measure on your own data before planning around a number.
|
|
204
224
|
```
|
|
205
225
|
|
|
206
226
|
### `:fixed_accuracy` — absolute error bound
|
|
@@ -209,10 +229,11 @@ The workhorse for financial and ML workloads. You set a maximum per-element erro
|
|
|
209
229
|
ZFP uses as few bits as needed to honor it.
|
|
210
230
|
|
|
211
231
|
```ruby
|
|
212
|
-
# Error on any element guaranteed ≤ tolerance
|
|
232
|
+
# Error on any finite element guaranteed ≤ tolerance (NaN/Infinity are not preserved in lossy modes)
|
|
213
233
|
bytes = Zfp.compress(prices, type: :double, shape: [n], mode: :fixed_accuracy, tolerance: 0.001)
|
|
214
234
|
# $0.001 per price point — indistinguishable for P&L calculations
|
|
215
|
-
#
|
|
235
|
+
# Measured: 2.7x–3.0x on simulated daily closes at tolerance 0.001;
|
|
236
|
+
# 2.8x → 4.8x on a sinusoid as tolerance rises from 0.001 to 1.0
|
|
216
237
|
```
|
|
217
238
|
|
|
218
239
|
### `:fixed_precision` — significant bits
|
|
@@ -222,7 +243,8 @@ an absolute error bound. Handy for scientific data where relative precision matt
|
|
|
222
243
|
|
|
223
244
|
```ruby
|
|
224
245
|
bytes = Zfp.compress(data, type: :double, shape: [n], mode: :fixed_precision, precision: 20)
|
|
225
|
-
#
|
|
246
|
+
# precision is the bit budget per transform coefficient (up to 64 for a double, 32 for a float);
|
|
247
|
+
# it is not a mantissa-bit count. Error is relative to the largest value in each 4-element block.
|
|
226
248
|
```
|
|
227
249
|
|
|
228
250
|
### `:fixed_rate` — guaranteed bytes per value
|
|
@@ -235,26 +257,26 @@ bytes = Zfp.compress(data, type: :double, shape: [n], mode: :fixed_rate, rate: 1
|
|
|
235
257
|
# 16 bits per value → 4x smaller than raw double
|
|
236
258
|
# WARNING: aggressive rates (< 8) can produce large errors on high-dynamic-range data.
|
|
237
259
|
# Always validate max_err against your tolerance before committing to a rate.
|
|
260
|
+
# NOTE: libzfp enforces a per-block minimum (about 3 bits/value for 1-D doubles); lower
|
|
261
|
+
# requests are rounded up and the output is sized by the rate actually applied.
|
|
238
262
|
```
|
|
239
263
|
|
|
240
264
|
---
|
|
241
265
|
|
|
242
266
|
## Real-World Performance
|
|
243
267
|
|
|
244
|
-
Benchmarked on
|
|
245
|
-
(synthetic GBM data,
|
|
268
|
+
Benchmarked on 5 portfolios of 14–29 securities × 252 trading days of close prices
|
|
269
|
+
(synthetic GBM data, seed 42, `examples/02_portfolio_performance.rb`):
|
|
246
270
|
|
|
247
271
|
| Mode | Ratio | Max Error |
|
|
248
272
|
|---|---|---|
|
|
249
|
-
| `:reversible` | 1.
|
|
250
|
-
| `:fixed_accuracy, tolerance: 0.001` |
|
|
251
|
-
| `:fixed_accuracy, tolerance: 0.01` | 3.4× | < $0.01 |
|
|
273
|
+
| `:reversible` | 1.0×–1.1× | 0 (exact) |
|
|
274
|
+
| `:fixed_accuracy, tolerance: 0.001` | 2.7×–3.0× | $0.00035 |
|
|
252
275
|
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
expect 4×–10× with `:fixed_accuracy` and a tolerance tuned to preserve cosine similarity.
|
|
276
|
+
Independent GBM paths are a hard case: there is no cross-security correlation for the 2-D
|
|
277
|
+
transform to exploit, and daily returns are pure noise. Real market data (correlated sectors,
|
|
278
|
+
macro moves) should do better, but measure before you plan around a number. We have not yet
|
|
279
|
+
benchmarked ML embedding vectors; the tolerance sweep in the docs shows how to measure your own.
|
|
258
280
|
|
|
259
281
|
---
|
|
260
282
|
|
|
@@ -262,15 +284,21 @@ expect 4×–10× with `:fixed_accuracy` and a tolerance tuned to preserve cosin
|
|
|
262
284
|
|
|
263
285
|
```ruby
|
|
264
286
|
Zfp::LibraryNotFound # libzfp not installed or not found
|
|
265
|
-
Zfp::
|
|
266
|
-
Zfp::
|
|
267
|
-
Zfp::
|
|
287
|
+
Zfp::NumoNotAvailable # Numo output requested but numo-narray is not installed
|
|
288
|
+
Zfp::InvalidType # unrecognized :type, non-Array data, or integer data out of range
|
|
289
|
+
Zfp::InvalidMode # unrecognized :mode, or a lossy mode with an integer type
|
|
290
|
+
Zfp::InvalidShape # bad shape, or shape product ≠ number of elements
|
|
268
291
|
Zfp::InvalidParams # missing or invalid mode-specific param (rate/precision/tolerance)
|
|
269
292
|
Zfp::CompressionFailed # libzfp returned an error during compress
|
|
270
|
-
Zfp::DecompressionFailed #
|
|
271
|
-
Zfp::PackerError # corrupt or
|
|
293
|
+
Zfp::DecompressionFailed # empty input, truncated :fixed_rate stream, or libzfp error
|
|
294
|
+
Zfp::PackerError # corrupt, truncated, or 0.1.0 multi-dimensional pack header
|
|
272
295
|
```
|
|
273
296
|
|
|
297
|
+
Decompressing with the wrong type, shape, mode, or parameters **cannot** be detected for raw
|
|
298
|
+
`compress` output: the bytes carry no header, so you get a full-size array of garbage, not an
|
|
299
|
+
error. The gem only guarantees that this is memory-safe. Use `pack`/`unpack` when the
|
|
300
|
+
parameters need to travel with the data.
|
|
301
|
+
|
|
274
302
|
All inherit from `Zfp::Error < StandardError`.
|
|
275
303
|
|
|
276
304
|
---
|
|
@@ -283,7 +311,7 @@ All inherit from `Zfp::Error < StandardError`.
|
|
|
283
311
|
- **ML embedding storage** — vectors stay geometrically faithful under `:fixed_accuracy` at
|
|
284
312
|
tolerances that preserve cosine similarity.
|
|
285
313
|
- **Message queues** — pack/unpack means the consumer needs no schema; bytes are self-describing.
|
|
286
|
-
- **Checkpointing numerical computation** — save intermediate Numo arrays between steps.
|
|
314
|
+
- **Checkpointing numerical computation** — `pack(...).save("step.zfp")` intermediate Numo arrays between steps; `Zfp.load` restores them with shape and type intact.
|
|
287
315
|
|
|
288
316
|
## When Not to Use This Gem
|
|
289
317
|
|
|
@@ -299,7 +327,7 @@ All inherit from `Zfp::Error < StandardError`.
|
|
|
299
327
|
|
|
300
328
|
```bash
|
|
301
329
|
bundle exec ruby examples/01_basic_usage.rb
|
|
302
|
-
bundle exec ruby examples/
|
|
330
|
+
bundle exec ruby examples/02_portfolio_performance.rb
|
|
303
331
|
```
|
|
304
332
|
|
|
305
333
|
---
|
|
@@ -309,7 +337,7 @@ bundle exec ruby examples/portfolio_performance.rb
|
|
|
309
337
|
- Ruby ≥ 3.0
|
|
310
338
|
- libzfp 1.0+ — install with [upkg](https://github.com/seuros/upkg): `upkg install zfp`
|
|
311
339
|
- [ffi](https://github.com/ffi/ffi) ~> 1.0 (installed automatically)
|
|
312
|
-
- [numo-narray](https://github.com/ruby-numo/numo-narray) _(optional —
|
|
340
|
+
- [numo-narray](https://github.com/ruby-numo/numo-narray) _(optional — required on demand for Numo output)_
|
|
313
341
|
|
|
314
342
|
---
|
|
315
343
|
|
|
@@ -343,4 +371,4 @@ Please include a failing test for bugs and a passing test for features.
|
|
|
343
371
|
|
|
344
372
|
## License
|
|
345
373
|
|
|
346
|
-
MIT. Go nuts.
|
|
374
|
+
MIT. Go nuts. See [LICENSE](LICENSE).
|
data/lib/zfp/codec.rb
CHANGED
|
@@ -2,13 +2,15 @@
|
|
|
2
2
|
|
|
3
3
|
module Zfp
|
|
4
4
|
class Codec
|
|
5
|
-
VALID_TYPES
|
|
6
|
-
VALID_MODES
|
|
5
|
+
VALID_TYPES = %i[float double int32 int64].freeze
|
|
6
|
+
VALID_MODES = %i[fixed_rate fixed_precision fixed_accuracy reversible].freeze
|
|
7
|
+
INTEGER_TYPES = %i[int32 int64].freeze
|
|
7
8
|
|
|
8
9
|
def initialize(type:, shape:, mode:, numo: false, **params)
|
|
9
10
|
validate_type!(type)
|
|
10
11
|
validate_mode!(mode)
|
|
11
|
-
|
|
12
|
+
validate_type_mode!(type, mode)
|
|
13
|
+
Field.validate_shape!(shape)
|
|
12
14
|
validate_params!(mode, params)
|
|
13
15
|
@type = type
|
|
14
16
|
@shape = shape
|
|
@@ -18,7 +20,7 @@ module Zfp
|
|
|
18
20
|
end
|
|
19
21
|
|
|
20
22
|
def compress(data)
|
|
21
|
-
buf = TypeCoercion.to_buffer(data, @type)
|
|
23
|
+
buf = TypeCoercion.to_buffer(data, @type, @shape)
|
|
22
24
|
field = Field.new(@type, @shape, buf)
|
|
23
25
|
Stream.new(@mode, @params).compress(field)
|
|
24
26
|
ensure
|
|
@@ -36,11 +38,13 @@ module Zfp
|
|
|
36
38
|
field&.free
|
|
37
39
|
end
|
|
38
40
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
+
# numo: overrides what Zfp.unpack will return later. By default a Numo
|
|
42
|
+
# input (or a codec built with numo: true) unpacks to Numo.
|
|
43
|
+
def pack(data, numo: nil)
|
|
44
|
+
as_numo = numo.nil? ? (@numo || TypeCoercion.numo?(data)) : numo
|
|
41
45
|
compressed = compress(data)
|
|
42
|
-
Packer.encode(compressed, type: @type, shape: @shape, mode: @mode,
|
|
43
|
-
|
|
46
|
+
Packed.new(Packer.encode(compressed, type: @type, shape: @shape, mode: @mode,
|
|
47
|
+
params: @params, numo: as_numo))
|
|
44
48
|
end
|
|
45
49
|
|
|
46
50
|
private
|
|
@@ -55,12 +59,12 @@ module Zfp
|
|
|
55
59
|
unless VALID_MODES.include?(mode)
|
|
56
60
|
end
|
|
57
61
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
62
|
+
# Integer types are lossless by nature; the gem supports them in
|
|
63
|
+
# :reversible mode only, as documented.
|
|
64
|
+
def validate_type_mode!(type, mode)
|
|
65
|
+
return unless INTEGER_TYPES.include?(type) && mode != :reversible
|
|
66
|
+
raise Zfp::InvalidMode,
|
|
67
|
+
"#{type} supports mode: :reversible only (got #{mode.inspect})"
|
|
64
68
|
end
|
|
65
69
|
|
|
66
70
|
def validate_params!(mode, params)
|
data/lib/zfp/errors.rb
CHANGED
data/lib/zfp/field.rb
CHANGED
|
@@ -4,8 +4,27 @@ module Zfp
|
|
|
4
4
|
class Field
|
|
5
5
|
attr_reader :zfp_type_int, :shape
|
|
6
6
|
|
|
7
|
+
# Validates that +shape+ is an Array of 1-4 positive Integers.
|
|
8
|
+
# Shared by Codec (at construction) and Field (at allocation).
|
|
9
|
+
def self.validate_shape!(shape)
|
|
10
|
+
unless shape.is_a?(Array) && shape.length.between?(1, 4) &&
|
|
11
|
+
shape.all? { |d| d.is_a?(Integer) && d > 0 }
|
|
12
|
+
raise Zfp::InvalidShape,
|
|
13
|
+
"shape must be an Array of 1–4 positive integers, got #{shape.inspect}"
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# Ruby Arrays and Numo::NArray are row-major: the LAST dimension varies
|
|
18
|
+
# fastest in memory. libzfp's nx is the FASTEST-varying dimension (see
|
|
19
|
+
# zfp FAQ #0), so the shape must be reversed before it is handed to zfp.
|
|
20
|
+
# Callers always work in row-major `shape:`; this is the only place the
|
|
21
|
+
# conversion happens.
|
|
22
|
+
def self.zfp_dimensions(shape)
|
|
23
|
+
shape.reverse
|
|
24
|
+
end
|
|
25
|
+
|
|
7
26
|
def initialize(type, shape, buffer)
|
|
8
|
-
validate_shape!(shape)
|
|
27
|
+
self.class.validate_shape!(shape)
|
|
9
28
|
@type = type
|
|
10
29
|
@shape = shape
|
|
11
30
|
@buffer = buffer # hold reference — prevents GC while field is alive
|
|
@@ -34,20 +53,13 @@ module Zfp
|
|
|
34
53
|
|
|
35
54
|
private
|
|
36
55
|
|
|
37
|
-
def validate_shape!(shape)
|
|
38
|
-
unless shape.is_a?(Array) && shape.length.between?(1, 4) &&
|
|
39
|
-
shape.all? { |d| d.is_a?(Integer) && d > 0 }
|
|
40
|
-
raise Zfp::InvalidShape,
|
|
41
|
-
"shape must be an Array of 1–4 positive integers, got #{shape.inspect}"
|
|
42
|
-
end
|
|
43
|
-
end
|
|
44
|
-
|
|
45
56
|
def set_shape(shape)
|
|
57
|
+
nx, ny, nz, nw = self.class.zfp_dimensions(shape)
|
|
46
58
|
case shape.length
|
|
47
|
-
when 1 then Zfp::FFI.zfp_field_set_size_1d(@ptr,
|
|
48
|
-
when 2 then Zfp::FFI.zfp_field_set_size_2d(@ptr,
|
|
49
|
-
when 3 then Zfp::FFI.zfp_field_set_size_3d(@ptr,
|
|
50
|
-
when 4 then Zfp::FFI.zfp_field_set_size_4d(@ptr,
|
|
59
|
+
when 1 then Zfp::FFI.zfp_field_set_size_1d(@ptr, nx)
|
|
60
|
+
when 2 then Zfp::FFI.zfp_field_set_size_2d(@ptr, nx, ny)
|
|
61
|
+
when 3 then Zfp::FFI.zfp_field_set_size_3d(@ptr, nx, ny, nz)
|
|
62
|
+
when 4 then Zfp::FFI.zfp_field_set_size_4d(@ptr, nx, ny, nz, nw)
|
|
51
63
|
end
|
|
52
64
|
end
|
|
53
65
|
end
|
data/lib/zfp/packed.rb
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Zfp
|
|
4
|
+
# A self-describing compressed array: the 32-byte pack header plus the
|
|
5
|
+
# libzfp payload, as produced by Zfp.pack / Codec#pack. The natural unit
|
|
6
|
+
# for persistence — a .zfp file is complete without a sidecar schema.
|
|
7
|
+
#
|
|
8
|
+
# packed = Zfp.pack(data, type: :double, shape: [252], mode: :reversible)
|
|
9
|
+
# packed.save("prices.zfp")
|
|
10
|
+
# Zfp.load("prices.zfp").unpack # => data
|
|
11
|
+
#
|
|
12
|
+
# Anything that accepts a String via #to_str (File.binwrite, Redis, Zfp.unpack)
|
|
13
|
+
# accepts a Packed directly.
|
|
14
|
+
class Packed
|
|
15
|
+
EXTENSION = ".zfp"
|
|
16
|
+
|
|
17
|
+
def self.load(path)
|
|
18
|
+
new(File.binread(path))
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def initialize(bytes)
|
|
22
|
+
@bytes = bytes.to_str.dup.freeze
|
|
23
|
+
Packer.decode_header(@bytes) # fail fast on bytes that are not a pack
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def to_str
|
|
27
|
+
@bytes
|
|
28
|
+
end
|
|
29
|
+
alias to_s to_str
|
|
30
|
+
|
|
31
|
+
def bytesize
|
|
32
|
+
@bytes.bytesize
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Header fields, read without decompressing.
|
|
36
|
+
def type = header[:type]
|
|
37
|
+
def shape = header[:shape]
|
|
38
|
+
def mode = header[:mode]
|
|
39
|
+
def params = header[:params]
|
|
40
|
+
def numo? = header[:numo]
|
|
41
|
+
|
|
42
|
+
def save(path)
|
|
43
|
+
File.binwrite(path, @bytes)
|
|
44
|
+
path
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def unpack
|
|
48
|
+
Zfp.unpack(@bytes)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def ==(other)
|
|
52
|
+
other.respond_to?(:to_str) && to_str == other.to_str
|
|
53
|
+
end
|
|
54
|
+
alias eql? ==
|
|
55
|
+
|
|
56
|
+
def hash
|
|
57
|
+
@bytes.hash
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def inspect
|
|
61
|
+
"#<Zfp::Packed type=#{type.inspect} shape=#{shape.inspect} mode=#{mode.inspect}" \
|
|
62
|
+
"#{params.empty? ? "" : " #{params.map { |k, v| "#{k}=#{v}" }.join(" ")}"}" \
|
|
63
|
+
"#{numo? ? " numo" : ""} bytesize=#{bytesize}>"
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
private
|
|
67
|
+
|
|
68
|
+
def header
|
|
69
|
+
@header ||= Packer.decode_header(@bytes)
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
data/lib/zfp/packer.rb
CHANGED
|
@@ -2,9 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
module Zfp
|
|
4
4
|
module Packer
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
# Header version 2 (zfp gem 0.2.0+): shape is row-major and the gem
|
|
6
|
+
# reverses it for libzfp. Version 1 (0.1.0) passed shape[0] to libzfp as
|
|
7
|
+
# nx, i.e. transposed, so multi-dimensional v1 streams cannot be decoded
|
|
8
|
+
# with the corrected layout. 1-D v1 streams are identical and still load.
|
|
9
|
+
MAGIC = "ZFP\x02"
|
|
10
|
+
LEGACY_MAGIC = "ZFP\x01"
|
|
11
|
+
HEADER_LEN = 32
|
|
12
|
+
PACK_FMT = "a4CCCCVVVVE"
|
|
8
13
|
|
|
9
14
|
TYPE_TO_BYTE = { float: 0, double: 1, int32: 2, int64: 3 }.freeze
|
|
10
15
|
BYTE_TO_TYPE = TYPE_TO_BYTE.invert.freeze
|
|
@@ -22,7 +27,17 @@ module Zfp
|
|
|
22
27
|
header + compressed_bytes
|
|
23
28
|
end
|
|
24
29
|
|
|
30
|
+
# => [type, shape, mode, params, numo, compressed_data]
|
|
25
31
|
def self.decode(bytes)
|
|
32
|
+
bytes = bytes.to_str
|
|
33
|
+
h = decode_header(bytes)
|
|
34
|
+
[h[:type], h[:shape], h[:mode], h[:params], h[:numo], bytes[HEADER_LEN..]]
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Parses only the 32-byte header, without touching the payload.
|
|
38
|
+
# Raises Zfp::PackerError if +bytes+ is not a pack this version can read.
|
|
39
|
+
def self.decode_header(bytes)
|
|
40
|
+
bytes = bytes.to_str
|
|
26
41
|
if bytes.bytesize < HEADER_LEN
|
|
27
42
|
raise Zfp::PackerError, "truncated header (#{bytes.bytesize} < #{HEADER_LEN} bytes)"
|
|
28
43
|
end
|
|
@@ -30,16 +45,28 @@ module Zfp
|
|
|
30
45
|
magic, type_b, mode_b, rank, flags, d0, d1, d2, d3, param =
|
|
31
46
|
bytes[0, HEADER_LEN].unpack(PACK_FMT)
|
|
32
47
|
|
|
33
|
-
|
|
48
|
+
validate_magic!(magic, rank)
|
|
34
49
|
|
|
35
|
-
type
|
|
36
|
-
mode
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
50
|
+
type = BYTE_TO_TYPE.fetch(type_b) { raise Zfp::PackerError, "unknown type byte #{type_b}" }
|
|
51
|
+
mode = BYTE_TO_MODE.fetch(mode_b) { raise Zfp::PackerError, "unknown mode byte #{mode_b}" }
|
|
52
|
+
{
|
|
53
|
+
type: type,
|
|
54
|
+
shape: [d0, d1, d2, d3].first(rank),
|
|
55
|
+
mode: mode,
|
|
56
|
+
params: decode_params(mode, param),
|
|
57
|
+
numo: flags & 1 == 1
|
|
58
|
+
}
|
|
59
|
+
end
|
|
41
60
|
|
|
42
|
-
|
|
61
|
+
private_class_method def self.validate_magic!(magic, rank)
|
|
62
|
+
return if magic == MAGIC
|
|
63
|
+
if magic == LEGACY_MAGIC
|
|
64
|
+
return if rank == 1
|
|
65
|
+
raise Zfp::PackerError,
|
|
66
|
+
"#{rank}-D data packed by zfp gem 0.1.0 used a transposed dimension order " \
|
|
67
|
+
"and cannot be unpacked by this version; unpack with 0.1.0 and re-pack"
|
|
68
|
+
end
|
|
69
|
+
raise Zfp::PackerError, "bad magic bytes (expected ZFP\\x02)"
|
|
43
70
|
end
|
|
44
71
|
|
|
45
72
|
private_class_method def self.extract_param(mode, params)
|
data/lib/zfp/stream.rb
CHANGED
|
@@ -9,7 +9,7 @@ module Zfp
|
|
|
9
9
|
|
|
10
10
|
def compress(field)
|
|
11
11
|
zfp_ptr = bitstream = output_buf = nil
|
|
12
|
-
zfp_ptr =
|
|
12
|
+
zfp_ptr = open_zfp_stream
|
|
13
13
|
apply_mode(zfp_ptr, field)
|
|
14
14
|
bufsize = Zfp::FFI.zfp_stream_maximum_size(zfp_ptr, field.pointer)
|
|
15
15
|
output_buf = ::FFI::MemoryPointer.new(:uint8, bufsize)
|
|
@@ -24,13 +24,23 @@ module Zfp
|
|
|
24
24
|
Zfp::FFI.zfp_stream_close(zfp_ptr) if zfp_ptr && !zfp_ptr.null?
|
|
25
25
|
end
|
|
26
26
|
|
|
27
|
+
# libzfp's bitstream has no bounds checking: it reads as many words as the
|
|
28
|
+
# field requires, regardless of how many bytes the caller supplied. The
|
|
29
|
+
# input is therefore copied into a zero-filled buffer at least as large as
|
|
30
|
+
# the worst-case stream size, so a short or corrupt input can never read
|
|
31
|
+
# past the end of the buffer.
|
|
27
32
|
def decompress(field, compressed_bytes)
|
|
28
33
|
zfp_ptr = bitstream = input_buf = nil
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
zfp_ptr
|
|
34
|
+
raise Zfp::DecompressionFailed, "no compressed bytes given" if compressed_bytes.bytesize.zero?
|
|
35
|
+
|
|
36
|
+
zfp_ptr = open_zfp_stream
|
|
32
37
|
apply_mode(zfp_ptr, field)
|
|
33
|
-
|
|
38
|
+
validate_input_size!(compressed_bytes.bytesize, field)
|
|
39
|
+
max_size = Zfp::FFI.zfp_stream_maximum_size(zfp_ptr, field.pointer)
|
|
40
|
+
bufsize = [max_size, compressed_bytes.bytesize].max
|
|
41
|
+
input_buf = ::FFI::MemoryPointer.new(:uint8, bufsize) # zero-filled
|
|
42
|
+
input_buf.put_bytes(0, compressed_bytes)
|
|
43
|
+
bitstream = Zfp::FFI.stream_open(input_buf, bufsize)
|
|
34
44
|
Zfp::FFI.zfp_stream_set_bit_stream(zfp_ptr, bitstream)
|
|
35
45
|
Zfp::FFI.zfp_stream_rewind(zfp_ptr)
|
|
36
46
|
read = Zfp::FFI.zfp_decompress(zfp_ptr, field.pointer)
|
|
@@ -40,13 +50,45 @@ module Zfp
|
|
|
40
50
|
Zfp::FFI.zfp_stream_close(zfp_ptr) if zfp_ptr && !zfp_ptr.null?
|
|
41
51
|
end
|
|
42
52
|
|
|
53
|
+
# The rate libzfp actually applied in the last fixed_rate call. libzfp
|
|
54
|
+
# rounds the requested rate up to its per-block minimum, so this can be
|
|
55
|
+
# higher than params[:rate]. nil for the other modes.
|
|
56
|
+
attr_reader :actual_rate
|
|
57
|
+
|
|
58
|
+
# Exact compressed size of a fixed_rate stream for +field+:
|
|
59
|
+
# blocks * bits_per_block, rounded up to a 64-bit stream word.
|
|
60
|
+
def fixed_rate_size(field)
|
|
61
|
+
blocks = field.shape.reduce(1) { |acc, d| acc * ((d + 3) / 4) }
|
|
62
|
+
bits_per_block = (@actual_rate * (4**field.dims)).round
|
|
63
|
+
words = (blocks * bits_per_block + 63) / 64
|
|
64
|
+
words * 8
|
|
65
|
+
end
|
|
66
|
+
|
|
43
67
|
private
|
|
44
68
|
|
|
69
|
+
def open_zfp_stream
|
|
70
|
+
ptr = Zfp::FFI.zfp_stream_open(nil)
|
|
71
|
+
raise Zfp::Error, "Failed to allocate zfp_stream" if ptr.null?
|
|
72
|
+
ptr
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# In fixed-rate mode the compressed size is fully determined by the shape
|
|
76
|
+
# and the applied rate, so a shorter input is provably truncated. The
|
|
77
|
+
# variable-rate modes have no such invariant; for them the zero padding in
|
|
78
|
+
# #decompress is the only protection.
|
|
79
|
+
def validate_input_size!(actual, field)
|
|
80
|
+
return unless @mode == :fixed_rate
|
|
81
|
+
expected = fixed_rate_size(field)
|
|
82
|
+
return if actual >= expected
|
|
83
|
+
raise Zfp::DecompressionFailed,
|
|
84
|
+
"fixed_rate stream is truncated: expected #{expected} bytes, got #{actual}"
|
|
85
|
+
end
|
|
86
|
+
|
|
45
87
|
def apply_mode(zfp_ptr, field)
|
|
46
88
|
case @mode
|
|
47
89
|
when :fixed_rate
|
|
48
|
-
Zfp::FFI.zfp_stream_set_rate(zfp_ptr, @params[:rate].to_f,
|
|
49
|
-
|
|
90
|
+
@actual_rate = Zfp::FFI.zfp_stream_set_rate(zfp_ptr, @params[:rate].to_f,
|
|
91
|
+
field.zfp_type_int, field.dims, 0)
|
|
50
92
|
when :fixed_precision
|
|
51
93
|
Zfp::FFI.zfp_stream_set_precision(zfp_ptr, @params[:precision])
|
|
52
94
|
when :fixed_accuracy
|
data/lib/zfp/type_coercion.rb
CHANGED
|
@@ -23,6 +23,11 @@ module Zfp
|
|
|
23
23
|
int64: 8
|
|
24
24
|
}.freeze
|
|
25
25
|
|
|
26
|
+
INTEGER_RANGE = {
|
|
27
|
+
int32: (-2**31)..(2**31 - 1),
|
|
28
|
+
int64: (-2**63)..(2**63 - 1)
|
|
29
|
+
}.freeze
|
|
30
|
+
|
|
26
31
|
class << self
|
|
27
32
|
def detect_type(data)
|
|
28
33
|
return nil unless numo?(data)
|
|
@@ -44,8 +49,30 @@ module Zfp
|
|
|
44
49
|
defined?(Numo::NArray) && data.is_a?(Numo::NArray)
|
|
45
50
|
end
|
|
46
51
|
|
|
47
|
-
|
|
48
|
-
|
|
52
|
+
# Loads numo-narray on demand. Raises Zfp::NumoNotAvailable if the gem
|
|
53
|
+
# is not installed, so callers never see a bare NameError.
|
|
54
|
+
def ensure_numo!
|
|
55
|
+
return if defined?(Numo::NArray)
|
|
56
|
+
begin
|
|
57
|
+
require "numo/narray"
|
|
58
|
+
rescue LoadError
|
|
59
|
+
raise Zfp::NumoNotAvailable,
|
|
60
|
+
"Numo output requested but the numo-narray gem is not installed (gem install numo-narray)"
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Flattens +data+ (Array or Numo::NArray) into a packed FFI buffer.
|
|
65
|
+
# When +shape+ is given, the element count must equal its product;
|
|
66
|
+
# a mismatch would otherwise make libzfp read or write past the buffer.
|
|
67
|
+
def to_buffer(data, type, shape = nil)
|
|
68
|
+
values = flatten_values(data)
|
|
69
|
+
validate_count!(values, shape) if shape
|
|
70
|
+
validate_integers!(values, type) if INTEGER_RANGE.key?(type)
|
|
71
|
+
fmt = PACK_FORMAT.fetch(type) { raise Zfp::InvalidType, "Unknown type: #{type}" }
|
|
72
|
+
packed = values.pack(fmt)
|
|
73
|
+
ptr = ::FFI::MemoryPointer.new(:uint8, packed.bytesize)
|
|
74
|
+
ptr.put_bytes(0, packed)
|
|
75
|
+
ptr
|
|
49
76
|
end
|
|
50
77
|
|
|
51
78
|
def from_buffer(ptr, type, shape, as_numo)
|
|
@@ -54,20 +81,25 @@ module Zfp
|
|
|
54
81
|
|
|
55
82
|
private
|
|
56
83
|
|
|
57
|
-
def
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
ptr.put_bytes(0, packed)
|
|
62
|
-
ptr
|
|
84
|
+
def flatten_values(data)
|
|
85
|
+
return data.to_a.flatten if numo?(data)
|
|
86
|
+
return data.flatten if data.is_a?(Array)
|
|
87
|
+
raise Zfp::InvalidType, "data must be an Array or Numo::NArray, got #{data.class}"
|
|
63
88
|
end
|
|
64
89
|
|
|
65
|
-
def
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
90
|
+
def validate_count!(values, shape)
|
|
91
|
+
expected = shape.reduce(:*)
|
|
92
|
+
return if values.length == expected
|
|
93
|
+
raise Zfp::InvalidShape,
|
|
94
|
+
"shape #{shape.inspect} describes #{expected} elements but data has #{values.length}"
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def validate_integers!(values, type)
|
|
98
|
+
range = INTEGER_RANGE[type]
|
|
99
|
+
bad = values.find { |v| !v.is_a?(Integer) || !range.cover?(v) }
|
|
100
|
+
return unless bad
|
|
101
|
+
raise Zfp::InvalidType,
|
|
102
|
+
"#{type} data must be Integers in #{range.min}..#{range.max}, got #{bad.inspect}"
|
|
71
103
|
end
|
|
72
104
|
|
|
73
105
|
def buffer_to_array(ptr, type, shape)
|
|
@@ -77,6 +109,7 @@ module Zfp
|
|
|
77
109
|
end
|
|
78
110
|
|
|
79
111
|
def buffer_to_numo(ptr, type, shape)
|
|
112
|
+
ensure_numo!
|
|
80
113
|
count = shape.reduce(:*)
|
|
81
114
|
bytes = ptr.read_bytes(count * ELEMENT_SIZE[type])
|
|
82
115
|
array = bytes.unpack(PACK_FORMAT[type])
|
data/lib/zfp/version.rb
CHANGED
data/lib/zfp.rb
CHANGED
|
@@ -7,6 +7,7 @@ require_relative "zfp/type_coercion"
|
|
|
7
7
|
require_relative "zfp/field"
|
|
8
8
|
require_relative "zfp/stream"
|
|
9
9
|
require_relative "zfp/packer"
|
|
10
|
+
require_relative "zfp/packed"
|
|
10
11
|
require_relative "zfp/codec"
|
|
11
12
|
|
|
12
13
|
module Zfp
|
|
@@ -20,15 +21,25 @@ module Zfp
|
|
|
20
21
|
Codec.new(type: type, shape: shape, mode: mode, numo: numo, **params).decompress(bytes)
|
|
21
22
|
end
|
|
22
23
|
|
|
23
|
-
|
|
24
|
+
# Returns a Zfp::Packed: self-describing bytes that #unpack, #save(path),
|
|
25
|
+
# and convert to a String via #to_str. numo: controls what unpack returns
|
|
26
|
+
# later; it defaults to true for Numo::NArray input and false for an Array.
|
|
27
|
+
def self.pack(data, type: nil, shape: nil, mode:, numo: nil, **params)
|
|
24
28
|
type ||= TypeCoercion.detect_type(data) or raise InvalidType, "type: required for Ruby Array input"
|
|
25
29
|
shape ||= TypeCoercion.detect_shape(data) or raise InvalidShape, "shape: required for Ruby Array input"
|
|
26
|
-
as_numo = TypeCoercion.numo?(data)
|
|
27
|
-
Codec.new(type: type, shape: shape, mode: mode, numo: as_numo, **params).pack(data)
|
|
30
|
+
as_numo = numo.nil? ? TypeCoercion.numo?(data) : numo
|
|
31
|
+
Codec.new(type: type, shape: shape, mode: mode, numo: as_numo, **params).pack(data, numo: as_numo)
|
|
28
32
|
end
|
|
29
33
|
|
|
34
|
+
# Decompresses packed bytes (a String or a Zfp::Packed) back to data.
|
|
30
35
|
def self.unpack(bytes)
|
|
31
36
|
type, shape, mode, params, numo, compressed = Packer.decode(bytes)
|
|
32
37
|
Codec.new(type: type, shape: shape, mode: mode, numo: numo, **params).decompress(compressed)
|
|
33
38
|
end
|
|
39
|
+
|
|
40
|
+
# Reads a file written with Zfp::Packed#save. Returns the Zfp::Packed with
|
|
41
|
+
# its metadata intact; call #unpack on it for the data.
|
|
42
|
+
def self.load(path)
|
|
43
|
+
Packed.load(path)
|
|
44
|
+
end
|
|
34
45
|
end
|
metadata
CHANGED
|
@@ -1,109 +1,128 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: zfp
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
|
-
- Dewayne VanHoozer
|
|
7
|
+
- Dewayne VanHoozer
|
|
8
8
|
bindir: bin
|
|
9
9
|
cert_chain: []
|
|
10
10
|
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
11
|
dependencies:
|
|
12
|
-
- !ruby/object:Gem::Dependency
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
- !ruby/object:Gem::
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: ffi
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
-
|
|
17
|
+
- ~>
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: "1.0"
|
|
20
|
+
type: :runtime
|
|
21
|
+
prerelease: false
|
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
+
requirements:
|
|
24
|
+
-
|
|
25
|
+
- ~>
|
|
26
|
+
- !ruby/object:Gem::Version
|
|
27
|
+
version: "1.0"
|
|
28
|
+
- !ruby/object:Gem::Dependency
|
|
29
|
+
name: numo-narray
|
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
|
31
|
+
requirements:
|
|
32
|
+
-
|
|
33
|
+
- ">="
|
|
34
|
+
- !ruby/object:Gem::Version
|
|
35
|
+
version: "0"
|
|
36
|
+
type: :development
|
|
37
|
+
prerelease: false
|
|
38
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
39
|
+
requirements:
|
|
40
|
+
-
|
|
41
|
+
- ">="
|
|
42
|
+
- !ruby/object:Gem::Version
|
|
43
|
+
version: "0"
|
|
44
|
+
- !ruby/object:Gem::Dependency
|
|
45
|
+
name: minitest
|
|
46
|
+
requirement: !ruby/object:Gem::Requirement
|
|
47
|
+
requirements:
|
|
48
|
+
-
|
|
49
|
+
- ">="
|
|
50
|
+
- !ruby/object:Gem::Version
|
|
51
|
+
version: "0"
|
|
52
|
+
type: :development
|
|
53
|
+
prerelease: false
|
|
54
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
55
|
+
requirements:
|
|
56
|
+
-
|
|
57
|
+
- ">="
|
|
58
|
+
- !ruby/object:Gem::Version
|
|
59
|
+
version: "0"
|
|
60
|
+
- !ruby/object:Gem::Dependency
|
|
61
|
+
name: debug_me
|
|
62
|
+
requirement: !ruby/object:Gem::Requirement
|
|
63
|
+
requirements:
|
|
64
|
+
-
|
|
65
|
+
- ">="
|
|
66
|
+
- !ruby/object:Gem::Version
|
|
67
|
+
version: "0"
|
|
68
|
+
type: :development
|
|
69
|
+
prerelease: false
|
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
71
|
+
requirements:
|
|
72
|
+
-
|
|
73
|
+
- ">="
|
|
74
|
+
- !ruby/object:Gem::Version
|
|
75
|
+
version: "0"
|
|
68
76
|
description: |
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
77
|
+
Compress and decompress arrays of float/double/int32/int64 in 1-4
|
|
78
|
+
dimensions using ZFP's fixed-rate, fixed-precision, fixed-accuracy,
|
|
79
|
+
or lossless reversible modes. Accepts Ruby Array and Numo::NArray.
|
|
72
80
|
email:
|
|
73
|
-
- dvanhoozer@gmail.com
|
|
81
|
+
- dvanhoozer@gmail.com
|
|
74
82
|
executables: []
|
|
75
83
|
extensions: []
|
|
76
84
|
extra_rdoc_files: []
|
|
77
85
|
files:
|
|
78
|
-
-
|
|
79
|
-
-
|
|
80
|
-
-
|
|
81
|
-
- lib/zfp
|
|
82
|
-
- lib/zfp/
|
|
83
|
-
- lib/zfp/
|
|
84
|
-
- lib/zfp/
|
|
85
|
-
- lib/zfp/
|
|
86
|
-
- lib/zfp/
|
|
87
|
-
- lib/zfp/
|
|
88
|
-
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
86
|
+
- CHANGELOG.md
|
|
87
|
+
- LICENSE
|
|
88
|
+
- README.md
|
|
89
|
+
- lib/zfp.rb
|
|
90
|
+
- lib/zfp/codec.rb
|
|
91
|
+
- lib/zfp/errors.rb
|
|
92
|
+
- lib/zfp/ffi.rb
|
|
93
|
+
- lib/zfp/field.rb
|
|
94
|
+
- lib/zfp/packed.rb
|
|
95
|
+
- lib/zfp/packer.rb
|
|
96
|
+
- lib/zfp/stream.rb
|
|
97
|
+
- lib/zfp/type_coercion.rb
|
|
98
|
+
- lib/zfp/version.rb
|
|
99
|
+
- sig/zfp.rbs
|
|
100
|
+
homepage: "https://github.com/madbomber/zfp"
|
|
101
|
+
licenses:
|
|
102
|
+
- MIT
|
|
103
|
+
metadata:
|
|
104
|
+
homepage_uri: "https://github.com/madbomber/zfp"
|
|
105
|
+
source_code_uri: "https://github.com/madbomber/zfp"
|
|
106
|
+
changelog_uri: "https://github.com/madbomber/zfp/blob/main/CHANGELOG.md"
|
|
107
|
+
documentation_uri: "https://madbomber.github.io/zfp"
|
|
108
|
+
bug_tracker_uri: "https://github.com/madbomber/zfp/issues"
|
|
92
109
|
rdoc_options: []
|
|
93
110
|
require_paths:
|
|
94
|
-
- lib
|
|
111
|
+
- lib
|
|
95
112
|
required_ruby_version: !ruby/object:Gem::Requirement
|
|
96
113
|
requirements:
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
114
|
+
-
|
|
115
|
+
- ">="
|
|
116
|
+
- !ruby/object:Gem::Version
|
|
117
|
+
version: 3.0.0
|
|
100
118
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
101
119
|
requirements:
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
120
|
+
-
|
|
121
|
+
- ">="
|
|
122
|
+
- !ruby/object:Gem::Version
|
|
123
|
+
version: "0"
|
|
105
124
|
requirements: []
|
|
106
|
-
rubygems_version: 4.0.
|
|
125
|
+
rubygems_version: 4.1.0.beta3
|
|
107
126
|
specification_version: 4
|
|
108
127
|
summary: Ruby FFI bindings for the ZFP floating-point compression library
|
|
109
128
|
test_files: []
|