cumo 0.10.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 78444d61470bd2cf534dfadba1facef1d92785cf85db498ca14d893242532591
-  data.tar.gz: 7bb662f78bd6e130c5a93db7fe6ac44fc2dad14f58f3788de48c04f7830197b4
+  metadata.gz: 48de893a0d01a5e8376c2a9cda402be28dd47835661f22bd69e7ed17561abed8
+  data.tar.gz: c403ffc7d25b80a0b50e8513b81439ee46f775cbd8c126667f83656077488a79
 SHA512:
-  metadata.gz: 46d001da3c8c82e4f835fe6643cec1a1f5a4b28864283286a52f98887d975673914ee475ecd90d97042e5c16e45c525060d595ecd8301a427fed5060a8efc7b0
-  data.tar.gz: ae0052bea91487469a2dd6052c08f9bb8c74799447ea165d189babc88ccd94b7c686e50d1ff69f1df75ec2d20986c7113ff2d3b949205c4ac72132f7b8d468af
+  metadata.gz: '00189a16920241771a70155a98d60c5d53b6fcd23bfcddbaed3094bbbe5d3476b2d6b5ce70f9d8d899e090a9bf3be2c4249c1b98b6d14360736a4a5ba1aa5594'
+  data.tar.gz: bd6a1c8b78837b387f93f5b57f13c26917c71c123a1441deff8e3afb7250a42bc799ded9271b9ad9d46d4c3b928660c38653f85427b024bd00797d6786535734

data/.yardopts

@@ -0,0 +1,10 @@
+--no-private
+--exclude ext/cumo/narray/gen/
+-e ext/cumo/narray/gen/cogen_runner.rb
+lib/**/*.rb
+ext/**/*.c
+ext/**/*.cpp
+tmp/yard/types/*.c
+-
+*.md
+LICENSE.txt

data/3rd_party/mkmf-cu/lib/mkmf-cu/cli.rb

@@ -38,7 +38,7 @@ module MakeMakefileCuda
     # TODO(sonots): Make it possible to configure "nvcc" and additional arguments
     def nvcc_command
       s = MakeMakefileCuda::Nvcc.generate(argv)
-      cmd = "nvcc #{s}"
+      cmd = "nvcc #{s} -Xfatbin=-compress-all"
       if ENV['CUMO_NVCC_GENERATE_CODE']
         ENV['CUMO_NVCC_GENERATE_CODE'].split.each do |code|
           cmd << " --generate-code=#{code}"

data/CHANGELOG.md

@@ -1,3 +1,62 @@
+# 0.12.0 (2026/10/04)
+
+Fixes:
+
+* Fix a stream left current in a thread that ended never being destroyed. Neither `Stream#destroy` nor the finalizer could free it, so about 0.5 MiB of device memory stayed behind per such stream, and under M:N threads the next Ruby thread on the same native thread inherited it. The current stream now lives on the Ruby `Thread`, which adds about 2% to the fixed cost of a launch (PR #593)
+* Fix a device memory leak each time a thread that used cuBLAS, cuSOLVER or cuDNN ends: 64 MiB per thread for cuBLAS, 32 MiB more for cuSOLVER and about 8 MiB for cuDNN. An ending thread now leaves its handles to the next one, so device memory stays at what the most threads alive at once needed (PR #591)
+* Fix `x.inplace.cumsum` and `cumprod` answering a new array and leaving `x` as it was, which they did since 0.10.0 (PR #582)
+
+Changes:
+
+* Add `Cumo::Linalg`, the functions of `Numo::Linalg` from numo-linalg-alt 0.10.1 on cuSOLVER and cuBLAS, with the same names, arguments, return values and errors: `blas_char`, `dot`, `matmul`, `matrix_power`, `solve`, `inv`, `det`, `slogdet`, `lu`, `lu_fact`, `lu_solve`, `lu_inv`, `cholesky`, `cho_fact`, `cho_solve`, `cho_inv`, `ldl`, `qr`, `eig`, `eigvals`, `eigh`, `eigvalsh`, `svd`, `svdvals`, `pinv`, `lstsq`, `matrix_rank`, `orth`, `null_space`, `norm`, `cond` and `expm`. Load it with `require "cumo/linalg"`. Cumo builds it when the toolkit has cuSOLVER's 64-bit API, and `Cumo::CUDA::Cusolver.available?` says whether it did. `eig` and `eigvals` need `cusolverDnXgeev`, which CUDA 11 does not have, and `ldl` of a Hermitian complex matrix needs `cusolverDnXhetrf`, which CUDA 11 and 12 do not have. Where it differs from numo-linalg-alt on purpose, as in `expm`, `norm` and `cond`, the README says how (PR #594, PR #592, PR #589, PR #588, PR #587, PR #586, PR #584, PR #581, PR #580, PR #577, PR #576, PR #575, PR #574, PR #573, PR #572, PR #571, PR #570, PR #569, PR #568, PR #567)
+* `require "cumo"` sets `CUDA_MODULE_LOADING` to `LAZY` unless it is set already, as PyTorch does. Drivers before 535 otherwise load every kernel module when CUDA starts, which adds about 2.2 GB of RSS. Child processes inherit the setting (PR #590)
+* `triu`, `tril`, `triu!` and `tril!` clear the triangle of a contiguous array with an Integer `k` in one kernel rather than through an index array: `triu` of a 16 x 16 DFloat takes 0.006 ms where it took 0.49 (PR #585)
+* The first kernel of a type costs less host memory. Each type builds separate modules for its reductions, their nan-aware forms and `mulsum` against another type, so DFloat adds 88 MB of RSS for elementwise work and 137 MB once it reduces, where its first kernel of any kind added 181 MB. Every fatbin image is compressed, which takes `cumo.so` from 120 MB to 112 MB (PR #583)
+* The API docs show the NArray methods defined in C, such as `shape` and `NArray.zeros`, and keep the docstrings of the typed-class methods. `.yardopts` keeps the build-time code generators out of the docs, and the gemspec names rubydoc.info as the documentation (PR #579, PR #578)
+* The gem no longer ships the test suite, `bench/` and the files for working on cumo (PR #566)
+
+# 0.11.0 (2026/09/27)
+
+Breaking changes:
+
+* `sort`, `sort_index`, `cumsum` and `cumprod` raise `ArgumentError` on `keepdims:`, which they accepted and ignored, since they keep the shape of their input. They take `axis:` and `nan:`, and `nan:` stays on every type as it does for `sum` and `median`. Numo still accepts `keepdims:` here without a word (PR #552)
+* `reverse` raises `ArgumentError` on `keepdims:` and `nan:`, which it accepted and ignored, and takes `axis:` only. Numo still accepts both without a word (PR #551)
+
+Fixes:
+
+* Fix a store from an overlapping part of the same array, such as `a[0...-k] = a[k..]`, writing wrong values, since threads read elements other threads had already overwritten. With 10 million SFloat elements and a shift of 1024 every run was wrong, on every path and every dtype. The source is now copied first when it shares an element with the destination, which gives numpy's answer (PR #553, PR #542)
+* Fix `ElementwiseKernel` and `ReductionKernel` racing when an input shares memory with an output but is not read element for element, such as a contiguous input shifted against its output. Such an input is copied first (PR #535)
+* Fix a leak of the new shape when `expand_dims` cannot allocate its strides (PR #563)
+
+Changes:
+
+* `cumsum` and `cumprod` down axis 0, or along a transposed view, gather and scatter their buffer through a shared-memory tile: on a 2048 x 2048 SFloat, `a.cumsum(axis: 0)` spends 133 us on the GPU where it spent 315, and the gather for `a.transpose.cumsum(axis: 1)` takes 51 us where it took 154 (PR #565)
+* An elementwise op on a view built on an index array gathers it into a buffer a typed element at a time rather than with a memcpy of a run-time size: `a[idx, true] + 1` on a 512 x 512 SFloat spends 2.75 us in the gather where it spent 5.2 (PR #564)
+* Say in the API docs of `MemoryPool.used_bytes` and `total_bytes` what each counts and when, and in the README that `used_bytes` waits for the garbage collector where CuPy's `used_bytes()` and PyTorch's `memory_allocated()` do not (PR #562)
+* `ElementwiseKernel` and `ReductionKernel` accept `Cumo::HFloat` and `Cumo::BFloat` as `float16` and `bfloat16`, CUDA's `__half` and `__nv_bfloat16`. A Ruby number handed to one is rounded on the host the way `Cumo::HFloat[x]` and `Cumo::BFloat[x]` round it (PR #561)
+* CI builds against CUDA 13.4.1 and 12.9.2 where it built against 13.2.1, and keeps 12.8.1 (PR #560)
+* `concatenate`, and `hstack`, `vstack` and `dstack` through it, write every part in one kernel launch when every part is a contiguous array of the result's class, and no longer fill the result with zeros first: `hstack` of six SFloat parts of 1500 x 64 takes 5.2 us where it took 18.3 (PR #559)
+* Elementwise ops, bias adds and math functions such as `gelu` move 16 bytes of each operand a thread when the last axis is contiguous in every operand: at 1500 x 384 and 1500 x 1536 SFloat, an add takes 4.18 us where it took 5.67, a bias add 15.1 where it took 27.4, and `gelu` 16.7 where it took 23.7. An HFloat add is 2.3 times faster and an Int8 add 3.6 times. Integer `div`, `mod` and `reciprocal`, and DComplex, keep the old loop (PR #558)
+* Say in the README and in comments why one spelling is faster than another rather than the timings of one GPU, which read as wrong on another card (PR #557, PR #556)
+* `layer_norm` and `rms_norm` hold a row in registers when it divides into sixteen-byte vectors and fits in eight of them a thread, and read it once: `layer_norm` over 1500 x 384 takes 6.7 us where it took 18.5. `layer_norm` takes the variance about the mean in a second pass, which is also more accurate, and a row whose sum overflows near the top of the range stays finite (PR #555)
+* `softmax` holds a row of up to 16 elements a thread in registers and reads it once: 1500 x 1500 takes 18.8 us where it took 34.4. SFloat and DFloat answers move by a few ulp, and HFloat and BFloat come closer to the reference, since `exp` is no longer rounded to half precision before the scale (PR #554)
+* `sort`, `sort_index`, `cumsum`, `cumprod` and `reverse` accept an empty array and answer an empty array of the same shape, where they raised. A missing axis still raises `DimensionError` (PR #550, PR #549)
+* A copy that the wide byte copy cannot take, as reductions, `argmax` and `sort` make of a non-contiguous input, goes a typed element at a time: `argmax` over a column slice runs 1.9 times faster and a copy of a column slice 1.3 times (PR #548)
+* A kernel over a view of 2 to 4 dimensions whose offsets fit an int32 does its address arithmetic in 32 bits: a store with a cast, `abs` or `clip` on such a view runs 1.17 to 1.32 times faster (PR #547)
+* A DFloat `logseq` of base 10 or 2 calls `exp10` or `exp2` rather than `pow`: 200,000 elements take 20 us where they took 84 (PR #546)
+* The rank switch of every generated kernel launcher comes from one helper in the kernel generator, and the generated sources are token for token what they were (PR #545)
+* `logseq` writes a strided, index or transposed view in one launch rather than one per row: a 6250 x 32 DFloat view takes 87 us where it took 14 ms (PR #544)
+* A reduction along a last axis of 16 elements or fewer gives a row that spans more than one 32-byte sector two threads: `Cumo::SFloat.ones(1536, 64)[true, 0...16].sum(axis: 1)` takes 2.18 us where it took 4.20, and `max`, `mean`, `mulsum` and `argmax` over short rows run 1.2 to 1.8 times faster (PR #543)
+* A store between views of the same type moves 16, 8, 4 or 2 bytes a thread over a run contiguous and aligned on both sides, through one set of kernels the copy behind `dup` shares: the store of an im2col unfold runs about 2 times faster (PR #541)
+* `cumsum` and `cumprod` no longer wait for the GPU. A 1-D scan shorter than 8192 elements ran on the host after a device synchronize, and every other shape waited for the stream to drain: `x = a * 1; x.cumsum + 1` takes about 9 us where it took 0.3 to 0.8 ms (PR #540)
+* `rand`, `rand_norm` and `seq` write a view built on an index array directly rather than through a buffer copied out and back, and so do `real=` and `imag=`: `rand` into a `[6250, 1024][true, idx64]` SFloat view takes 12.7 us where it took 80.7. An axis of length one merges with its neighbour, so a `[N, 1]` column runs what a flat array runs. A seed draws the same values as before (PR #539)
+* `ElementwiseKernel` and `ReductionKernel` read a reversed or stepped view, or a frozen array, where it is rather than copying it on every call, and `ReductionKernel` reads a transposed view the same way: a reduction along the first axis of a transposed 4000 by 4000 array no longer pays a copy that took longer than the reduction. An `ElementwiseKernel` input that is transposed is still copied, in tiles, which beats reading it across rows (PR #538, PR #535)
+* `ReductionKernel` over a leading axis gives a block up to 32 outputs that lie end to end, so a warp reads neighbouring outputs: a sum over axis 0 of a [4000, 4000] SFloat takes 0.169 ms where it took 0.413, what Cumo's own `sum` takes (PR #537)
+* `ElementwiseKernel` and `ReductionKernel` walk neighbouring dimensions that every array lays out end to end as one: a `ReductionKernel` sum over everything of a [160, 100, 1000] SFloat takes 0.162 ms where it took 0.193 (PR #536)
+* Fix a stream test that failed about once in a thousand runs, since its fill on a non-blocking stream was not ordered before its launch on the null stream (PR #534)
+* Say in the README where `CUMO_ALLOW_TF32` pays and where it does not: it speeds up matrix products and convolutions, buys nothing where reading the weights takes the time, and its rounding can push a training step past a tolerance single precision meets (PR #533)
+* A number stored with `a[...] = x`, `a[] = x` or `store(x)` goes through `fill`, one kernel launch with nothing reserved, where it filled a 0-dimensional array first. `fill` writes a view built on an index array or a mask directly rather than through a copy of the whole view (PR #532)
+
 # 0.10.0 (2026/09/22)
 
 Breaking changes:

data/README.md

@@ -10,6 +10,7 @@ Cumo (pronounced "koomo") is a CUDA-aware, GPU-optimized numerical library that
 * NVIDIA GPU Compute Capability 3.5 (Kepler) or later
 * CUDA 11.0 or later
 * cuDNN 8.0 or later (optional, for the cuDNN features)
+* cuSOLVER with its 64-bit API, which comes with the CUDA Toolkit (optional, for `Cumo::Linalg`)
 
 ## Preparation
 
@@ -185,21 +186,11 @@ The 0-dimensional return is what lets an iterative loop stay on the GPU.
 Reading a scalar back to the host waits for everything queued behind it, so every read caps how far ahead the GPU is allowed to run.
 What a read costs is not a fixed price either: it is however much work happens to be queued when it is taken.
 
-`bench/cg_bench.rb` prices this with a conjugate gradient solve, 200 iterations over a 512x512 grid on an RTX 5070 Ti Laptop:
-
-```
-scalars        convergence test    us/iter    readbacks/iter
-Ruby Floats    every iteration       136.7        2.02
-Ruby Floats    never                 134.4        2.02
-0-dim NArray   every iteration       142.6        1.02
-0-dim NArray   every 20th             65.6        0.06
-0-dim NArray   never                  60.3        0.02
-```
-
+`bench/cg_bench.rb` measures this with a conjugate gradient solve over a 512x512 grid, with the scalars kept either way and the convergence test taken every iteration, every twentieth or never.
 Written with Ruby Floats the loop reads back twice an iteration whatever the convergence test does, since `alpha` needs `pap` and `beta` needs `rs_new` as Floats.
-Thinning the test cannot get under that floor, and keeping the scalars as 0-dimensional NArrays buys nothing on its own.
-The two only pay together, and together they are worth 2.1x.
-The relative residual is identical in every row.
+Thinning the test cannot get under that floor, and keeping the scalars as 0-dimensional NArrays buys nothing on its own, since the test still reads back every iteration.
+The two only pay together, and together they let the loop run well ahead of the host.
+The relative residual is identical either way.
 
 ```ruby
 alpha = rs_old / pap   # a 0-dimensional NArray, divided on the device
@@ -208,8 +199,8 @@ x += p_dir * alpha     # and consumed there, without crossing the bus
 
 Read the value back once the loop is done, or every k iterations if it has to test something.
 
-A read costs the wait and a copy of the block it needs into pinned host memory, a few microseconds for a scalar.
-Reading managed memory from the host directly would fault its page over instead, and a small block shares a page with other live blocks that the next kernel touches, so a fresh scalar cost 0.8 ms that way on the machine above, forty times what the copy costs.
+A read costs the wait and a copy of the block it needs into pinned host memory.
+Reading managed memory from the host directly would fault its page over instead, and a small block shares a page with other live blocks that the next kernel touches, so a fresh scalar costs far more that way than the copy does.
 The reads that answer values, from `Float(x)` and `to_a` to `each` and `inspect`, take the copy.
 
 ### Reshape Copies, Reshape! Does Not
@@ -226,18 +217,7 @@ a.reshape(2, 12)[0, 0] = 99.0  # a copy, so a does not
 This is what Numo does too, and numpy is where the expectation comes from: there `reshape` answers a view whenever the strides allow one.
 On a GPU the difference is an allocation and a copy kernel, paid every call.
 `reshape!` changes the receiver in place and costs neither.
-RTX 5070 Ti Laptop, `Cumo::SFloat`, 200 calls a measurement:
-
-```
-shape        reshape     reshape!    the copy allocates
-1x768        5.1 us      0.04 us          3 KB
-1024x768    39.1 us      0.18 us          3 MB
-4096x768    58.1 us      0.22 us         12 MB
-8192x768   142.7 us      0.43 us         24 MB
-```
-
-`reshape!` is host-side bookkeeping, so it stays under a microsecond whatever the array weighs.
-The `reshape` column is the copy, and it grows with the bytes.
+It is host-side bookkeeping, so its cost does not depend on what the array weighs, where the copy `reshape` makes grows with the bytes.
 
 The catch is that `reshape!` changes the array everything else is holding.
 It fits a temporary the calling expression owns, and not an argument, an ivar, or anything a cache still points at:
@@ -262,28 +242,17 @@ a + 2.0                     #=> Cumo::SFloat
 ```
 
 Numo promotes the same way, and on a CPU it costs nothing: Numo's single-precision math computes in double and narrows the result anyway.
-On a GeForce card, whose double-precision rate is a sixty-fourth of its single-precision one, it costs a great deal.
-512x2048 elements in place on an RTX 5070 Ti Laptop:
-
-```
-                SFloat     DFloat
-a * 2.0         11.8 us    12.8 us
-sqrt            11.0 us    43.0 us
-sin             11.5 us    97.5 us
-atan            11.1 us   122.9 us
-atan2           12.5 us   192.4 us
-```
-
-Only the transcendentals pay for the promotion; a double multiply runs at the speed of a single one.
+On a GeForce card, whose double-precision rate is a small fraction of its single-precision one, it costs a great deal.
+Only the transcendentals pay for the promotion, and the more work the function does the more they pay: a double multiply runs at about the speed of a single one, while `sqrt`, `sin`, `atan` and `atan2` in double take several times as long as in single.
 The methods a Float can reach as a second argument are `atan2`, `hypot` and `ldexp`.
-`ldexp` pays a different way, since scaling by a power of two is cheap in either precision: `Cumo::NMath.ldexp(a, 2.0)` takes 268.0 us against 12.6 us for `Cumo::NMath.ldexp(a, 2)`, and the difference there is the doubled arrays it has to allocate rather than the arithmetic.
+`ldexp` pays a different way, since scaling by a power of two is cheap in either precision: `Cumo::NMath.ldexp(a, 2.0)` is still far slower than `Cumo::NMath.ldexp(a, 2)`, and the difference there is the doubled arrays it has to allocate rather than the arithmetic.
 
 Pass a 0-dimensional array instead of a Float and the call stays single precision.
 That is what `[]` hands back, so a scalar taken out of an array is already in the right form:
 
 ```ruby
 two = Cumo::SFloat[2.0][0]     # a 0-dimensional Cumo::SFloat
-Cumo::NMath.atan2(a, two)      #=> Cumo::SFloat, 14.9 us against 219.3 us
+Cumo::NMath.atan2(a, two)      #=> Cumo::SFloat
 ```
 
 Naming the module directly works too, under both libraries:
@@ -300,22 +269,9 @@ The 0-dimensional form has no effect under Numo, where `[]` returns a Ruby Float
 That saves the copy and the memory it needs.
 What it costs is the kernel cuBLAS then picks, which for most shapes is slower than the one it picks for an operand already laid out its way.
 
-RTX 5070 Ti Laptop, `Cumo::SFloat`, `q[M,K].dot(k[N,K].transpose)`, medians of nine rounds:
-
-```
-M     K     N       M*K      as it is   copied first   the copy
-1     64    1500         64      5.6 us       11.4 us      2.8 us
-512   64    512      32,768      8.5         11.2         2.9
-1500  64    1500     96,000     33.0         29.3         3.0
-512   256   512     131,072     21.8         20.7         2.8
-4096  64    4096    262,144    229.1        179.3         4.2
-512   768   512     393,216     54.0         39.2         3.3
-256   3072  768     786,432    131.1        103.0        12.5
-```
-
 The copy weighs `N * K`, and the faster kernel it buys is worth `M * N * K`, so `M` is what decides.
-A matrix-vector product, where `M` is one, is the clearest case against copying: it takes twice as long that way.
-Past an `M * K` of roughly fifty thousand on this card the copy starts paying for itself, and past a few hundred thousand it is worth a fifth of the time.
+A matrix-vector product, where `M` is one, is the clearest case against copying.
+As `M * K` grows the copy starts paying for itself; where it starts depends on the card, so measure the shape that matters before choosing.
 
 Where a profile says one of these multiplications matters, hand it an operand that is already contiguous:
 
@@ -324,13 +280,47 @@ kt = k.transpose.dup    # or build k transposed in the first place
 q.dot(kt)
 ```
 
-The table above is two-dimensional, where cuBLAS is given one matrix.
-A batched multiplication takes another path through the same flag, and these numbers do not cover it.
+All of this is about the two-dimensional case, where cuBLAS is given one matrix.
+A batched multiplication takes another path through the same flag.
+
+### Linear Algebra
+
+`require "cumo/linalg"` brings in `Cumo::Linalg`, the functions of `Numo::Linalg` from [numo-linalg-alt](https://github.com/yoshoku/numo-linalg-alt) on the GPU.
+They keep numo-linalg-alt's names, arguments, return values and errors, and run on cuSOLVER and cuBLAS.
+
+```ruby
+require "cumo/linalg"
+
+a = Cumo::DFloat.new(3, 3).rand + Cumo::DFloat.eye(3) * 3
+b = Cumo::DFloat.new(3).rand
+x = Cumo::Linalg.solve(a, b)
+w, v = Cumo::Linalg.eigh(a + a.transpose)
+```
+
+The functions are `blas_char`, `dot`, `matmul`, `matrix_power`, `solve`, `inv`, `det`, `slogdet`, `lu`, `lu_fact`, `lu_solve`, `lu_inv`, `cholesky`, `cho_fact`, `cho_solve`, `cho_inv`, `ldl`, `qr`, `eig`, `eigvals`, `eigh`, `eigvalsh`, `svd`, `svdvals`, `pinv`, `lstsq`, `matrix_rank`, `orth`, `null_space`, `norm`, `cond` and `expm`.
+They take `SFloat`, `DFloat`, `SComplex` and `DComplex`.
+An integer or `Bit` matrix is computed in `DFloat`, as numo-linalg-alt does, and `HFloat` and `BFloat` raise `TypeError`.
+
+Cumo looks for cuSOLVER when it is built.
+Without it, the functions that need it raise `NotImplementedError`, and `Cumo::CUDA::Cusolver.available?` answers whether it is there.
+`eig` and `eigvals` also need `cusolverDnXgeev`, which CUDA 11 does not have.
+`ldl` of a Hermitian complex matrix needs `cusolverDnXhetrf`, which CUDA 11 and 12 do not have.
+
+Cumo differs from numo-linalg-alt in a few places on purpose:
+
+* Where numo-linalg-alt answers wrong, Cumo answers as numpy does: a wide matrix in `orth` and `lstsq`, a zero matrix in `pinv`, a matrix of full rank in `null_space`, an integer matrix in `qr` and `svd`, and empty matrices.
+* `expm` scales and squares the way scipy does, after Al-Mohy and Higham, which stays accurate where the norm of the matrix is large. Passing the order of the Pade approximant, as in `expm(a, 3)`, keeps numo-linalg-alt's algorithm.
+* `norm` takes the 2-norm and the nuclear norm of matrices stacked along other axes, which numo-linalg-alt refuses.
+* `cond` checks `ord` before it inverts the matrix, so an invalid `ord` raises `ArgumentError` even for a singular matrix, where numo-linalg-alt raises `LapackError`.
+
+A function waits for the current stream when it answers a Ruby number, or when it reads back whether cuSOLVER could do its work.
+`solve`, `inv` and `det` read back whether the matrix was singular, `cho_fact` whether it was positive definite, and `eigh` and `svd` whether they converged.
+`matmul`, `dot` of two matrices, `matrix_power` to a positive power, `cholesky`, `cho_solve`, `qr` and `norm` along an axis wait for nothing, so a run of them is queued without the host waiting.
 
 ### Fused Operations
 
 `layer_norm`, `rms_norm` and `softmax` normalize along the last axis in one kernel each, and `quantize_symmetric` takes it to 8-bit integers in one more.
-Written out of the operators they take nine launches, six, five and six, and a launch costs about two microseconds whatever it is handed, so a short row pays for the launches rather than for its bytes.
+Written out of the operators they take nine launches, six, five and six, and a launch has a fixed cost whatever it is handed, so a short row pays for the launches rather than for its bytes.
 
 ```ruby
 y = x.layer_norm(gamma, beta, eps: 1e-5)   # (x - mean) / sqrt(var + eps) * gamma + beta
@@ -351,47 +341,8 @@ numpy, torch            -2    -2     0     0     2     2
 
 `rint` is the other rule, at the cost of writing the quantization out: `(x / scale[false, :new]).rint.clip(-127, 127)` answers what numpy answers. A tie needs `x` to be an exact odd multiple of half the scale, so whether one ever comes up is a property of the data rather than of the arithmetic. Over a million random single-precision elements the two spellings disagreed four times, and every disagreement was a tie.
 
-On an RTX 5070 Ti Laptop, against the same arithmetic spelled with operators, in microseconds:
-
-```
-layer_norm              SFloat            DFloat            HFloat
-shape              fused  written    fused  written    fused  written
-1 x 768              3.9     30.1      9.7     30.3      3.9     29.8
-256 x 768            6.6     22.5     22.0     35.4      7.8     22.4
-4096 x 768          32.7    177.4    300.7    610.5     28.2    142.9
-1 x 1000000         19.0     59.4    125.4    175.4     22.3     68.5
-
-softmax                 SFloat            DFloat            HFloat
-shape              fused  written    fused  written    fused  written
-1 x 768              4.3     19.8      7.4     26.5      6.3     34.4
-256 x 768            8.8     13.9     23.4     35.1      6.8     18.5
-4096 x 768          49.2    122.7    318.3    549.0     28.2    109.8
-1 x 1000000         27.8     44.0    108.6    148.5     32.1     53.8
-
-rms_norm                SFloat            DFloat            HFloat
-shape              fused  written    fused  written    fused  written
-1 x 768              2.9     14.2      5.8     14.1      3.1     15.1
-256 x 768            4.6     16.7      6.9     20.2      5.2     15.5
-4096 x 768          38.5    122.5    130.8    367.0     19.2     80.8
-1 x 1000000         12.5     35.8     23.7    102.2     12.0     33.2
-```
-
-```
-quantize_symmetric      SFloat            DFloat            HFloat
-shape              fused  written    fused  written    fused  written
-1 x 768              3.7     20.2      6.0     15.7      2.3     11.9
-256 x 768            4.7     15.0     20.2     21.3      4.7     13.0
-4096 x 768          24.4    105.4    289.5    333.3     23.8     80.3
-1 x 1000000         18.8     40.8     97.1     81.6     15.7     37.5
-```
-
-The tables were taken in separate sessions, so read each row against the row beside it and not across the tables.
-
-`quantize_symmetric` in `Cumo::DFloat` is worth less than the others and loses outright on a million elements, whatever shape they are in. The division it does per element is what costs: on this card one row of a million takes 44.0 us to divide in double against 10.8 in single, where the reduction over the same row takes 25.3 and 15.4. Six kernels give that division a kernel of its own to fill the device with, and one kernel leaves it behind the reduction.
-
-All three pay off in every precision, by the most where the row is short enough that the launches were all it was doing, and by the least in double, where the reduction itself costs more than the launches ever did.
-The memory clock on this card steps between 9001 and 11001 MHz under a benchmark this short, and the absolute figures move with it.
-The ratios hold across the steps.
+Against the same arithmetic spelled with operators they are faster, by the most where the row is short enough that the launches were all it was doing, and by the least in double, where the arithmetic itself costs more than the launches ever did.
+The exception is `quantize_symmetric` in `Cumo::DFloat`, which can lose on a long row. The division it does per element is what costs, since double is slow on a card with a low double-precision rate: six kernels give that division a kernel of its own to fill the device with, and one kernel leaves it behind the reduction.
 
 All three normalize along the last axis only.
 Other axes are reachable through `transpose`.
@@ -447,17 +398,7 @@ It is also called Swish.
 Cumo::NMath.silu(x)   # x / (1 + exp(-x)), in one kernel rather than five
 ```
 
-Written out of the operators it costs five launches, and one kernel runs 1.1x to 4.3x faster on an RTX 5070 Ti Laptop, in microseconds:
-
-```
-silu                    SFloat            DFloat            HFloat
-elements           fused  written    fused  written    fused  written
-768                  3.6     13.4      3.6      9.5      3.4     14.3
-786432               7.0     25.9     90.5    102.4      5.2     19.2
-16777216           353.2   1312.9   1984.7   3258.8    199.6    851.8
-```
-
-The three rows are three regimes rather than one curve: 768 elements pay for the launches, 786432 fit in L2 and move faster than this card reads from memory, and 16777216 are what it costs from DRAM.
+Written out of the operators it costs five launches, and one kernel is faster in every precision, by the most on an array small enough that the launches were all it was doing.
 
 It follows `torch.nn.functional.silu`, answering within one unit in the last place of the true value at the nine double points measured, as torch does.
 At the edges it matches torch exactly: `-Float::INFINITY` answers `NaN`, since that is what infinity times zero is, and a large negative `x` answers a signed zero.
@@ -474,15 +415,7 @@ Half reaches it at -21, because the round back to half gets there first.
 Cumo::NMath.sigmoid(x)   # in one kernel rather than four
 ```
 
-Written out of the operators it costs four launches, and one kernel runs 1.2x to 4.8x faster on an RTX 5070 Ti Laptop, in microseconds:
-
-```
-sigmoid                 SFloat            DFloat            HFloat
-elements           fused  written    fused  written    fused  written
-768                  2.9      9.0      2.5      8.8      2.0      9.6
-786432               5.6     23.0     94.6    111.3      7.3     23.6
-16777216           343.5   1234.7   1978.5   2941.1    210.5    798.1
-```
+Written out of the operators it costs four launches, and one kernel is faster in every precision, by the most on an array small enough that the launches were all it was doing.
 
 The kernel keeps the exponent's argument negative, which the plain quotient does not: writing `1 / (1 + exp(-x))` out asks `exp` for a value it cannot hold once `x` is negative enough, and the quotient then answers a zero where the curve is still a number the type carries.
 
@@ -513,17 +446,8 @@ A non-negative `x` takes the plain quotient unchanged, bit for bit, so only the
 Cumo::NMath.softplus(x)   # log(1 + exp(x)), in one kernel rather than three
 ```
 
-Written out of the operators it costs three launches, and one kernel runs 1.0x to 3.1x faster on an RTX 5070 Ti Laptop, in microseconds:
-
-```
-softplus                SFloat            DFloat            HFloat
-elements           fused  written    fused  written    fused  written
-768                  2.0      6.1      2.3      6.2      2.1      6.0
-786432               5.9     18.3    166.6    171.8      5.7     15.0
-16777216           344.2    939.5   3503.7   3938.2    206.2    591.2
-```
-
-Double barely moves at the two larger sizes, the arithmetic rather than the launches being what it pays for there.
+Written out of the operators it costs three launches, and one kernel is faster, by the most on an array small enough that the launches were all it was doing.
+Double gains little on a large array, the arithmetic rather than the launches being what it pays for there.
 
 Writing it out also gives out earlier than the kernel does, because the intermediate is an array of the receiver's type:
 
@@ -603,54 +527,22 @@ a.gemm(b, alpha: 0.001)    #=> 102.375, the true 102400 scaled down
 
 #### What half is faster at
 
-`gemm` reaches the tensor cores. Square matrices on an RTX 5070 Ti Laptop, median of three runs each:
-
-```
-              HFloat              SFloat              DFloat
-1024x1024     0.043 ms  49.7 TF   0.161 ms  13.3 TF   5.370 ms  0.40 TF
-2048x2048     0.362 ms  47.4 TF   1.360 ms  12.6 TF   43.46 ms  0.40 TF
-4096x4096     3.335 ms  41.2 TF   10.40 ms  13.2 TF   328.1 ms  0.42 TF
-```
+`gemm` reaches the tensor cores, which single and double precision do not, so a large matrix product runs several times faster in half than in single.
 
 An odd number of columns costs half far more than it costs the others, because a row then starts on a two-byte boundary and the vectorized path is gone.
-It is the column count of either operand that matters, not the row count.
-1024x1024 times 1024x1024, with one dimension made odd at a time:
-
-```
-             all even   M odd      K odd      N odd
-HFloat        51.8 TF   49.2 TF    23.3 TF    23.3 TF
-SFloat        11.9 TF      -       11.6 TF    10.8 TF
-```
-
-The run-to-run spread on these is a few per cent and reaches fifteen at the top end, so the M column says the row count does not matter rather than that it costs 5 per cent.
+It is the column count of either operand that matters, not the row count: making `K` or `N` odd loses a large share of the speed, where making `M` odd does not.
 
 The penalty is on the arithmetic, so it does not reach a matrix-vector product, which is bound by how fast the matrix can be read whatever its shape.
-A 1x768 by 768x50257 gemv takes 0.211 ms with that odd 50257 and 0.205 ms with 50256, a difference inside the noise; the same 768x50257 matrix against 256 rows takes 0.833 ms and 0.481 ms, which is not.
+A 1x768 by 768x50257 gemv runs as fast with that odd 50257 as with 50256, while the same matrix against 256 rows does not.
 Pad the inner dimensions of a real matrix product; leave a gemv alone.
 
 `conv` is a different story, and worth reading before reaching for half in a network.
 cuDNN chooses its algorithm from the ones that fit in a scratch buffer, and the half algorithms that use the tensor cores ask for a lot of it.
-The default ceiling reaches them; the 8MB one Cumo used to ship does not.
-N=32, C=K=64, 56x56, 3x3:
-
-```
-                                         HFloat     BFloat     SFloat
-CUMO_CUDNN_MAX_WORKSPACE_SIZE=8388608    1.27 ms    1.26 ms    1.19 ms
-unset (128MB)                            0.54 ms    0.60 ms    1.19 ms
-CUMO_CUDNN_MAX_WORKSPACE_SIZE=268435456  0.52 ms    0.59 ms    0.59 ms
-```
-
-This shape's single-precision algorithm wants more than the default, which is the other half of the reason to look at the ceiling for a network that spends its time in `conv`.
+The default ceiling reaches them; the 8MB one Cumo used to ship does not, and under it a half convolution such as N=32, C=K=64, 56x56, 3x3 runs no faster than single.
+The single-precision algorithm for that shape wants more than the default, which is the other half of the reason to look at the ceiling for a network that spends its time in `conv`.
 
 Tensor cores also want the channel counts to be multiples of eight, which the first layer of a network never satisfies.
-That layer is still faster in half, but for the other reason:
-
-```
-C=3, K=64, 56x56, 3x3     0.085 ms   0.170 ms
-C=K=64, 56x56, 1x1        0.037 ms   0.120 ms
-```
-
-Neither of those reaches a tensor core; they move half the bytes.
+That layer can still be faster in half, but for the other reason: it moves half the bytes rather than reaching a tensor core.
 
 #### Batch normalization takes single-precision parameters
 
@@ -713,23 +605,8 @@ What replaces it is the mantissa: a bfloat16 resolves about three decimal digits
 
 #### What bfloat16 is faster at
 
-`gemm` reaches the tensor cores and lands on the same throughput as half. Square matrices on an RTX 5070 Ti Laptop, median of five runs each, one process per case:
-
-```
-              BFloat              HFloat              SFloat
-1024x1024     0.044 ms  48.9 TF   0.044 ms  48.5 TF   0.144 ms  14.9 TF
-2048x2048     0.298 ms  57.6 TF   0.303 ms  56.7 TF   1.404 ms  12.2 TF
-4096x4096     2.839 ms  48.4 TF   2.898 ms  47.4 TF   9.540 ms  14.4 TF
-```
-
+`gemm` reaches the tensor cores and runs at the same speed as half.
 The odd-column penalty is the same as half's and for the same reason, a row starting on a two-byte boundary.
-1024x1024 times 1024x1024, with one dimension made odd at a time:
-
-```
-             all even   M odd      K odd      N odd
-BFloat        48.9 TF   48.7 TF    23.5 TF    23.0 TF
-HFloat        48.5 TF      -       24.0 TF    23.1 TF
-```
 
 So the choice between the two sixteen-bit types is about range and resolution, not speed.
 Take bfloat16 where the magnitudes came from somewhere else and binary16 where three more mantissa bits are worth having.
@@ -739,7 +616,7 @@ Take bfloat16 where the magnitudes came from somewhere else and binary16 where t
 cuBLAS supports `CUDA_R_16BF`, which is what `gemm` and `dot` reach, from compute capability 8.0.
 Storing, casting and elementwise arithmetic have no such floor, because every operation is computed in single precision and rounded back, and they build and run wherever cumo does.
 A `dot` on a pre-Ampere card is the one to expect trouble from.
-**This is not measured here**: the only GPU these numbers came from is a Blackwell one, and the requirement is read from cuBLAS's documentation rather than reproduced.
+**This is not measured here**: the requirement is read from cuBLAS's documentation rather than reproduced on a card older than Ampere.
 
 cuDNN reaches bfloat16 as `CUDNN_DATA_BFLOAT16`, and its own bfloat16 kernels want Ampere for the same reason cuBLAS does.
 A convolution is given `CUDNN_DATA_FLOAT` to accumulate in, so it passes 256 the way a reduction does.
@@ -879,13 +756,19 @@ squared_diff.call(x, y)   # => the (2, 5) array of squared differences
 squared_diff.call(x, 5)   # => the same against a scalar
 ```
 
-A type is one of `float64`, `float32`, `int64`, `int32`, `int16`, `int8`, `uint64`, `uint32`, `uint16` and `uint8`, or a single letter that stands for whichever dtype the argument has.
+A type is one of `float64`, `float32`, `float16`, `bfloat16`, `int64`, `int32`, `int16`, `int8`, `uint64`, `uint32`, `uint16` and `uint8`, or a single letter that stands for whichever dtype the argument has.
 Outputs decide a letter before inputs do, and a letter that only a Ruby number reaches becomes `int64` or `float64`.
 Array arguments are broadcast against each other, and an output may be given after the inputs, or is allocated.
+An input is read through its own strides, so a reversed or stepped view goes to the kernel as it is.
+A transposed view is copied first, since the copy reorders it in tiles where the kernel would read it a row apart per thread, and so is a view built on an index array, or an input that shares memory with an output other than element for element, since the kernel would read what another thread has already written; a raw argument has to be contiguous and is copied when it is not.
 An argument marked `raw T y` is handed over as a pointer for the operation to index itself, with `i` the element index and `_ind.size()` the element count, and when every argument is raw or a number, `size:` says how many elements there are.
 Inputs are `const`, so an operation that writes one does not compile, and a number handed to an integer type has to be an Integer that fits.
 `preamble:` is placed before the kernel, after the typedefs of the letters, so a device function can be written in terms of `T`.
-`Cumo::Bit`, `Cumo::HFloat`, `Cumo::BFloat`, `Cumo::SComplex`, `Cumo::DComplex` and `Cumo::RObject` cannot be handed to one of these kernels yet.
+`float16` and `bfloat16` are CUDA's `__half` and `__nv_bfloat16`, compiled with the toolkit's `cuda_fp16.h` and `cuda_bf16.h`, which are found through `CUDA_PATH` or the `nvcc` on `PATH`.
+They convert to and from every arithmetic type, so an operation that mixes one with another type or a literal does not compile until one side is cast: `y = x * T(2)`, `y = x > T(0) ? x : T(0)` or `y = (float)x * 2`.
+With CUDA 11, arithmetic on `float16` does not compile for a GPU below compute capability 5.3, which has none.
+A number handed to one is rounded to it the way `Cumo::HFloat[x]` and `Cumo::BFloat[x]` round it.
+`Cumo::Bit`, `Cumo::SComplex`, `Cumo::DComplex` and `Cumo::RObject` cannot be handed to one of these kernels yet.
 
 ### Writing A Reduction Kernel
 
@@ -940,7 +823,8 @@ Cumo::CUDA::MemoryPool.total_bytes   # what the pool has taken from the card
 Cumo::CUDA::MemoryPool.used_bytes    # what it has handed out and not taken back
 ```
 
-`used_bytes` still counts blocks whose last reference is gone but which Ruby's garbage collector has not reached, so read it after `GC.start` when you mean the arrays that are alive:
+`used_bytes` still counts blocks whose last reference is gone but which Ruby's garbage collector has not reached, so read it after `GC.start` when you mean the arrays that are alive.
+Python frees an array when its last reference goes, so CuPy's `used_bytes()` and PyTorch's `memory_allocated()` drop at once where this waits for the collector:
 
 ```ruby
 a = Cumo::SFloat.new(1024, 1024).seq
@@ -1084,14 +968,8 @@ export CUMO_SHOW_WARNING_ONCE=OFF
 cuDNN picks a convolution algorithm by benchmarking the ones that fit in a scratch buffer, and the ceiling on that buffer is 128MB.
 The search reserves the whole ceiling whatever the convolution's size and hands it back to the pool afterwards, so the ceiling costs a peak rather than a residency.
 
-Some shapes want more than the default. The twenty convolutions of a ResNet-18 forward pass at batch 16, and the single-precision case from [Half Precision](#half-precision):
-
-```
-                                         ResNet-18    N=32, C=K=64, 56x56, 3x3
-CUMO_CUDNN_MAX_WORKSPACE_SIZE=8388608      9.80 ms                     1.19 ms
-unset (128MB)                              5.23 ms                     1.19 ms
-CUMO_CUDNN_MAX_WORKSPACE_SIZE=268435456    5.24 ms                     0.59 ms
-```
+The old 8MB ceiling kept the twenty convolutions of a ResNet-18 forward pass at batch 16 off their faster algorithms, which the default reaches.
+Some shapes want more than the default, such as the single-precision case from [Half Precision](#half-precision), which reaches its faster algorithm only above it.
 
 To raise it:
 
@@ -1116,20 +994,19 @@ To trade the accuracy for the speed:
 export CUMO_ALLOW_TF32=1
 ```
 
-Measured over the convolutions of a ResNet-18 forward pass at batch 16, with the ceiling raised to 256MB, each layer against a double precision reference:
-
-```
-                              pass      worst layer
-tensor cores off (default)   5.24 ms      1.4e-05
-tensor cores on              4.31 ms      2.4e-04
-```
+Over the convolutions of a ResNet-18 forward pass at batch 16, with the ceiling raised to 256MB, the pass gets faster and the worst layer moves from 1.4e-05 to 2.4e-04 against a double precision reference.
 
 The same flag puts `SFloat` and `SComplex` `gemm` on the tensor cores as TF32, and `dot` where it goes through `gemm`.
+`Cumo::Linalg.matrix_power` and `Cumo::Linalg.expm` multiply through `gemm`, so the flag reaches them too.
 Off, the answer is the one cuBLAS gives at single precision, bit for bit.
-On, a `[4096, 4096]` `SFloat` `gemm` goes from 17.8 to 28.4 TFLOP/s on an RTX 5070 Ti Laptop, 1.60 times over six interleaved rounds.
-Its answer is then about 3e-04 from a double precision reference, where it was 4e-07.
+On, a large `SFloat` `gemm` runs faster, and its answer is then about 3e-04 from a double precision reference, where it was 4e-07.
 The double types are not affected either way.
 
+What the flag buys depends on where the time goes.
+It pays where matrix products and convolutions take the time, as in training or in a batch through a convolutional network.
+It buys nothing where reading the operands takes the time, as in decoding one token at a time, where each step reads every weight once and the GEMM waits on memory rather than on arithmetic.
+Over a training run the rounding adds up, so a loss that meets a tolerance at single precision can miss it on some steps; compare a few steps with the flag off before relying on it.
+
 `Cumo.allow_tf32?` reads back the value in force.
 
 ## Contributing

data/cumo.gemspec

@@ -18,6 +18,7 @@ Gem::Specification.new do |spec|
 
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["documentation_uri"] = "https://rubydoc.info/gems/cumo/"
   spec.metadata["changelog_uri"] = "https://github.com/sonots/cumo/blob/master/CHANGELOG.md"
   spec.metadata["bug_tracker_uri"] = "https://github.com/sonots/cumo/issues"
   spec.metadata["rubygems_mfa_required"] = "true"
@@ -25,9 +26,8 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = ">= 3.0.0"
 
   spec.files = `git ls-files -z`.split("\x0").reject do |f|
-    f.match(%r{^(.github|test|spec|features|docker|examples)/})
+    f.start_with?(*%w[.github/ test/ examples/ bin/ bench/ docker- Dockerfile Gemfile .gitignore .rubocop])
   end
-  spec.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]

data/ext/cumo/cuda/cublas.c

@@ -43,31 +43,22 @@ cumo_cuda_cublas_check_status(cublasStatus_t status)
     }
 }
 
-// Lazily initialize cublas handle, and cache it
+static cumo_cuda_thread_local_t handles;
+
+// One handle per thread: the stream is set on the handle before each call,
+// and another thread with another current stream must not reset it in
+// between. A thread that ends leaves its handle to the next one.
 cublasHandle_t
 cumo_cuda_cublas_handle()
 {
-    // One table per thread: the stream is set on the handle before each call,
-    // and another thread with another current stream must not reset it in
-    // between. A handle is never destroyed.
-    static __thread cublasHandle_t *handles = 0;
-    int device;
-    if (handles == 0) {
-        int i;
-        int device_count = cumo_cuda_runtime_get_device_count();
-        handles = ALLOC_N(cublasHandle_t, device_count);
-        for (i = 0; i < device_count; ++i) {
-            handles[i] = 0;
-        }
-    }
-    device = cumo_cuda_runtime_get_device();
-    if (handles[device] == 0) {
+    cublasHandle_t *handle = (cublasHandle_t*)cumo_cuda_thread_local_get(&handles);
+    if (*handle == 0) {
         // A discarded status leaves the handle NULL, and cuBLAS reports that as
         // CUBLAS_STATUS_NOT_INITIALIZED at the next call instead of the reason.
-        cumo_cuda_cublas_check_status(cublasCreate(&handles[device]));
+        cumo_cuda_cublas_check_status(cublasCreate(handle));
     }
-    cumo_cuda_cublas_check_status(cublasSetStream(handles[device], cumo_cuda_stream()));
-    return handles[device];
+    cumo_cuda_cublas_check_status(cublasSetStream(*handle, cumo_cuda_stream()));
+    return *handle;
 }
 
 VALUE
@@ -88,6 +79,8 @@ Init_cumo_cuda_cublas(void)
     VALUE mCumo = rb_define_module("Cumo");
     VALUE mCUDA = rb_define_module_under(mCumo, "CUDA");
 
+    cumo_cuda_thread_local_init(&handles, sizeof(cublasHandle_t));
+
     /*
       Document-module: Cumo::Cublas
     */