cumo 0.14.0 → 0.14.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f09e6c09816ea16f1a27425e44170c7ea191b771f0a3b26b2b7298a1e53b6e6c
-  data.tar.gz: f4739b41e92d5f5920ea694393f3ab71241e9b428a29416db71e91d5fcb0f13e
+  metadata.gz: '0872662f2f59cb4486df50f2f9b6dee5fc62fe938f6c47bafdb1ebdfc744c2f6'
+  data.tar.gz: 248f4ea1224ee0ff46069d16b424b1c90a98d81b33610506c58d45ffb9693c08
 SHA512:
-  metadata.gz: d58332736efabf28caeec996882ceb99435885e41d017a40911c6a85346a4f03428e32cb7cae32e8dae9633e47f7497dc35075309ec32d61d607504ef5f61fb1
-  data.tar.gz: d276cc4b4d19203ab34bfae56e96dd6e3220a780f07d3da8934e9c9241336fcb31c2040e013a7e461b47ec93461dcff21ca126597742c184e52221c0112bf196
+  metadata.gz: caf8d608d673afd9e71d3c421d2c096db503bb0054bf1ca846f33de738ccb2662ab53a6c715e6003f9a27b48d148d1e8f60f509076d53eee1b1bebf56c023e97
+  data.tar.gz: 78c2fd433b79741516f842b069adfa90c0452694a106554e99fd0ac5415684d6b82beb7f44dae63a09a745fc075bf21c1fb71276fa8ad691965f9ddd335e8d9a

data/CHANGELOG.md

@@ -1,8 +1,21 @@
+# 0.14.1 (2026/10/22)
+
+Fixes:
+
+* Fix a host loop such as `each` yielding the old values of an array after its block wrote the array through `store_binary`, an in-place `cumsum` or `cumprod`, or an in-place `map` or `map_with_index`, also one paused by an Enumerator (PR #631, PR #630)
+* Fix an in-place `map_with_index` over a view that reaches an element more than once, such as `x[[0, 0, 0]]`, reading the element as it was before its own write. It now reads its own write, as `map` and numo do (PR #631)
+* Fix the release gem recording the tag of the previous release as `Cumo::REVISION`, as the 0.14.0 gem does with `"v0.13.0-11-gafd8ead9"`. `rake release` now makes the tag before it builds the gem, so this gem gives `"v0.14.1"` (PR #626)
+
+Changes:
+
+* `+`, `-`, `*` and `/` between a complex array and a real array of the same precision, in either order, read the real array as it is rather than casting it to a complex array first. On 4M elements this makes SComplex with SFloat about 1.7 to 1.8 times faster, and DComplex divided by DFloat about 1.4 times. The answers are the same but for the sign of a NaN (PR #632)
+* A host loop such as `each` whose block writes to the device copies the rest of the row, or of the array, again at once when copying the elements one at a time would cost more. With one write early in an `each_with_index` over 100,000 elements, the loop is about 90 times faster and about as fast as with no write. With a write at every 100th element it is about 12 times faster, and with a write at every other element about 16% slower (PR #631)
+
 # 0.14.0 (2026/10/18)
 
 Breaking changes:
 
-* `cumo.h` no longer defines `CUMO_VERSION` or `CUMO_VERSION_CODE`. `Cumo::VERSION` and `Cumo::NArray::VERSION` are defined in Ruby, and the new `Cumo::REVISION` tells a release gem from one built from a checkout: it gives `"v0.14.0"` for this release and something like `"v0.14.0-3-g1a2b3c4"` for a gem built at a later commit (PR #617)
+* `cumo.h` no longer defines `CUMO_VERSION` or `CUMO_VERSION_CODE`. `Cumo::VERSION` and `Cumo::NArray::VERSION` are defined in Ruby. The new `Cumo::REVISION` tells a release gem from one built from a checkout. A release gives its tag, and a gem built at a later commit gives something like `"v0.14.0-3-g1a2b3c4d"`. The 0.14.0 gem was built before its tag was made, so it gives `"v0.13.0-11-gafd8ead9"`. `rake release` makes the tag before the gem from the next release on (PR #617, PR #626)
 
 Fixes:
 

data/Rakefile

@@ -2,6 +2,35 @@
 
 require "bundler/gem_tasks"
 require "rake/testtask"
+require "rubygems/package"
+
+Rake::Task["release"].clear
+desc "Tag the version, build the gem at the tag, and push the branch, the tag and the gem"
+task :release, [:remote] do |_, args|
+  git = ->(*argv) { IO.popen(["git", *argv], err: File::NULL, &:read).strip }
+  version = File.read(File.join(__dir__, "lib/cumo/version.rb"))[/VERSION = "([^"]+)"/, 1]
+  tag = "v#{version}"
+  gem_file = "pkg/cumo-#{version}.gem"
+  remote = args[:remote] || "origin"
+  raise "There are files that need to be committed first." unless system("git", "diff", "--quiet") && system("git", "diff-index", "--quiet", "--cached", "HEAD")
+
+  tagged = git.call("rev-parse", "-q", "--verify", "#{tag}^{commit}")
+  raise "#{tag} is on #{tagged[0, 8]}, not on HEAD." unless tagged.empty? || tagged == git.call("rev-parse", "HEAD")
+
+  sh "git", "tag", "-m", "Version #{version}", tag if tagged.empty?
+  begin
+    mkdir_p "pkg"
+    sh "gem", "build", "cumo.gemspec", "--output", gem_file
+    revision = Gem::Package.new(gem_file).spec.metadata["cumo_revision"]
+    raise "#{gem_file} records #{revision.inspect} as cumo_revision, not #{tag.inspect}." unless revision == tag
+  rescue StandardError
+    sh "git", "tag", "-d", tag if tagged.empty?
+    raise
+  end
+  sh "git", "push", remote, "HEAD"
+  sh "git", "push", remote, "refs/tags/#{tag}"
+  sh "gem", "push", gem_file unless ENV["gem_push"] == "no"
+end
 
 Rake::TestTask.new(:test) do |t|
   t.libs << "test"

data/ext/cumo/include/cumo/cuda/runtime.h

@@ -33,10 +33,11 @@ extern size_t cumo_cuda_sync_epoch;
 // and reads again once the count moves.
 extern size_t cumo_cuda_launch_epoch;
 
-static inline void
+// Answers the count before the write.
+static inline size_t
 cumo_cuda_runtime_note_device_write(void)
 {
-    cumo_atomic_size_fetch_add(&cumo_cuda_launch_epoch, 1);
+    return cumo_atomic_size_fetch_add(&cumo_cuda_launch_epoch, 1);
 }
 
 // A pinned host buffer for reading device memory back: a copy into pinned

data/ext/cumo/include/cumo/ndloop.h

@@ -104,6 +104,13 @@ typedef struct {
 // it will read next again if device memory was written since it was staged.
 void cumo_na_ndloop_refresh_next(cumo_na_loop_t *lp, const void *next, size_t bytes);
 
+// For a function that yields to Ruby and writes an array that was there
+// before it, which cumo_na_ndloop_writes_existing answers: counts the element
+// it just wrote, so that a loop it yields to before it ends reads it. A staged
+// copy of its own has to hold the element already.
+int cumo_na_ndloop_writes_existing(cumo_na_loop_t *lp);
+void cumo_na_ndloop_wrote(cumo_na_loop_t *lp);
+
 // spec of user function
 typedef struct {
     cumo_na_iter_func_t func;    // user function

data/ext/cumo/narray/gen/tmpl/binary.c

@@ -10,6 +10,13 @@
 
 <% unless type_name == 'robject' %>
 void <%="cumo_#{c_iter}_kernel_launch"%>(cumo_na_iarray_t* a1, cumo_na_iarray_t* a2, cumo_na_iarray_t* a3, cumo_na_indexer_t* indexer, int* divzero);
+//<% if is_complex && %w[add sub mul div].include?(name) %>
+void <%="cumo_#{c_iter}_r_kernel_launch"%>(cumo_na_iarray_t* a1, cumo_na_iarray_t* a2, cumo_na_iarray_t* a3, cumo_na_indexer_t* indexer, int real_is_left);
+//<% end %>
+//<% complex_of = {'sfloat' => ['scomplex', 'cumo_cSComplex'], 'dfloat' => ['dcomplex', 'cumo_cDComplex']}[type_name] if %w[add sub mul div].include?(name) %>
+//<% if complex_of %>
+VALUE <%="cumo_#{complex_of[0]}_#{name}_real_left"%>(VALUE real, VALUE complex);
+//<% end %>
 //<% if %w[add sub mul div mod bit_and bit_or bit_xor left_shift right_shift copysign].include?(name) %>
 void <%="cumo_#{c_iter}_s_kernel_launch"%>(cumo_na_iarray_t* a1, dtype sv, cumo_na_iarray_t* a3, cumo_na_indexer_t* indexer, int* divzero, int scalar_is_left);
 //<% end %>
@@ -171,6 +178,40 @@ static void
 //<% end %>
 #undef check_intdivzero
 
+//<% if is_complex && %w[add sub mul div].include?(name) %>
+// The operands stay in the order they were written in, so that a shape
+// mismatch names them as it always did.
+static void
+<%=c_iter%>_r(cumo_na_loop_t *const lp)
+{
+    int real_is_left = *(int*)(lp->opt_ptr);
+    cumo_na_iarray_t a1 = cumo_na_make_iarray(&lp->args[real_is_left ? 1 : 0]);
+    cumo_na_iarray_t a2 = cumo_na_make_iarray(&lp->args[real_is_left ? 0 : 1]);
+    cumo_na_iarray_t a3 = cumo_na_make_iarray(&lp->args[2]);
+    cumo_na_indexer_t indexer = cumo_na_make_indexer(&lp->args[0]);
+
+    <%="cumo_#{c_iter}_r_kernel_launch"%>(&a1,&a2,&a3,&indexer,real_is_left);
+}
+
+// A real operand of this precision is read as it is rather than cast to a
+// complex array first.
+static VALUE
+<%=c_func%>_real(VALUE lhs, VALUE rhs, int real_is_left)
+{
+    cumo_ndfunc_arg_in_t ain[2] = {{real_is_left ? cRT : cT,0},{real_is_left ? cT : cRT,0}};
+    cumo_ndfunc_arg_out_t aout[1] = {{cT,0}};
+    cumo_ndfunc_t ndf = { <%=c_iter%>_r, CUMO_STRIDE_LOOP|CUMO_NDF_INDEXER_LOOP|CUMO_NDF_ANY_ORDER, 2, 1, ain, aout };
+
+    return cumo_na_ndloop3(&ndf, &real_is_left, 2, lhs, rhs);
+}
+
+VALUE
+<%="cumo_#{type_name}_#{name}_real_left"%>(VALUE real, VALUE self)
+{
+    return <%=c_func%>_real(real, self, 1);
+}
+//<% end %>
+
 static VALUE
 <%=c_func%>_self(VALUE self, VALUE other)
 {
@@ -202,6 +243,11 @@ static VALUE
         return cumo_na_ndloop3(&ndf_s, &y, 1, self);
     }
     //<% end %>
+    //<% if is_complex && %w[add sub mul div].include?(name) %>
+    if (rb_obj_class(other) == cRT) {
+        return <%=c_func%>_real(self, other, 0);
+    }
+    //<% end %>
     cumo_ndfunc_arg_in_t ain[2] = {{cT,0},{cT,0}};
     cumo_ndfunc_arg_out_t aout[1] = {{cT,0}};
     <% if type_name == 'robject' %>
@@ -230,6 +276,10 @@ static VALUE
     klass = cumo_na_upcast(rb_obj_class(self),rb_obj_class(other));
     if (klass==cT) {
         return <%=c_func%>_self(self, other);
+    //<% if complex_of %>
+    } else if (rb_obj_class(other) == <%=complex_of[1]%> && rb_obj_class(self) == cT) {
+        return <%="cumo_#{complex_of[0]}_#{name}_real_left"%>(self, other);
+    //<% end %>
     } else {
         v = rb_funcall(klass, cumo_id_cast, 1, self);
         return rb_funcall(v, <%=cumo_id_op%>, 1, other);

data/ext/cumo/narray/gen/tmpl/binary_kernel.cu

@@ -214,5 +214,29 @@ void <%="cumo_#{c_iter}_kernel_launch"%>(cumo_na_iarray_t* a1, cumo_na_iarray_t*
     cumo_cuda_runtime_check_kernel_launch();
 }
 //<% end %>
+
+//<% if is_complex && %w[add sub mul div].include?(name) %>
+// A real operand of the same precision is made complex as it is read, which is
+// what casting it first did, without the array the cast wrote.
+<% indexer_dims(true).each do |idim| %>
+__global__ void <%="cumo_#{c_iter}_r_kernel_dim#{idim}"%>(cumo_na_iarray_t a1, cumo_na_iarray_t a2, cumo_na_iarray_t a3, cumo_na_indexer_t indexer, int real_is_left)
+{
+    for (uint64_t i = blockIdx.x * blockDim.x + threadIdx.x; i < indexer.total_size; i += blockDim.x * gridDim.x) {
+        cumo_na_indexer_set_dim<%=idim%>(&indexer, i);
+        dtype x = *(dtype*)(cumo_na_iarray_at_dim<%=idim%>(&a1, &indexer));
+        dtype y = m_from_real(*(rtype*)(cumo_na_iarray_at_dim<%=idim%>(&a2, &indexer)));
+        *(dtype*)(cumo_na_iarray_at_dim<%=idim%>(&a3, &indexer)) = real_is_left ? m_<%=name%>(y, x) : m_<%=name%>(x, y);
+    }
+}
+<% end %>
+
+void <%="cumo_#{c_iter}_r_kernel_launch"%>(cumo_na_iarray_t* a1, cumo_na_iarray_t* a2, cumo_na_iarray_t* a3, cumo_na_indexer_t* indexer, int real_is_left)
+{
+    size_t grid_dim = cumo_get_grid_dim(indexer->total_size);
+    size_t block_dim = cumo_get_block_dim(indexer->total_size);
+    <%= indexer_switch("cumo_#{c_iter}_r_kernel", "*a1,*a2,*a3,*indexer,real_is_left", narrow: %w[a1 a2 a3]) %>
+    cumo_cuda_runtime_check_kernel_launch();
+}
+//<% end %>
 #undef cumo_check_intdivzero
 <% end %>

data/ext/cumo/narray/gen/tmpl/map_with_index.c

@@ -16,6 +16,12 @@ static void
     size_t *c;
     int nd, md;
 
+    // In place, a view that visits an element twice reads what it wrote the
+    // first time, so the copy the elements are read from takes the write too,
+    // after the element read next was copied again.
+    int own = lp->args[0].value == lp->args[1].value;
+    int counts = cumo_na_ndloop_writes_existing(lp);
+
     c = (size_t*)(lp->opt_ptr);
     cumo_na_with_index_dims(lp->ndim, &nd, &md);
     a = ALLOCA_N(VALUE,md);
@@ -30,40 +36,52 @@ static void
     if (idx1) {
         if (idx2) {
             for (; i--;) {
+                char *q = p1 + *idx1;
                 CUMO_GET_DATA_INDEX(p1,idx1,dtype,x);
                 x = yield_map_with_index(x,c,a,nd,md);
                 if (i) cumo_na_ndloop_refresh_next(lp, p1 + *idx1, sizeof(dtype));
                 if (cumo_cuda_runtime_sync_if_busy()) { CUMO_SHOW_SYNCHRONIZE_WARNING_ONCE("<%=name%>", "<%=type_name%>"); }
+                if (own) *(dtype*)q = x;
                 CUMO_SET_DATA_INDEX(p2,idx2,dtype,x);
+                if (counts) cumo_na_ndloop_wrote(lp);
                 c[nd]++;
             }
         } else {
             for (; i--;) {
+                char *q = p1 + *idx1;
                 CUMO_GET_DATA_INDEX(p1,idx1,dtype,x);
                 x = yield_map_with_index(x,c,a,nd,md);
                 if (i) cumo_na_ndloop_refresh_next(lp, p1 + *idx1, sizeof(dtype));
                 if (cumo_cuda_runtime_sync_if_busy()) { CUMO_SHOW_SYNCHRONIZE_WARNING_ONCE("<%=name%>", "<%=type_name%>"); }
+                if (own) *(dtype*)q = x;
                 CUMO_SET_DATA_STRIDE(p2,s2,dtype,x);
+                if (counts) cumo_na_ndloop_wrote(lp);
                 c[nd]++;
             }
         }
     } else {
         if (idx2) {
             for (; i--;) {
+                char *q = p1;
                 CUMO_GET_DATA_STRIDE(p1,s1,dtype,x);
                 x = yield_map_with_index(x,c,a,nd,md);
                 if (i) cumo_na_ndloop_refresh_next(lp, p1, sizeof(dtype));
                 if (cumo_cuda_runtime_sync_if_busy()) { CUMO_SHOW_SYNCHRONIZE_WARNING_ONCE("<%=name%>", "<%=type_name%>"); }
+                if (own) *(dtype*)q = x;
                 CUMO_SET_DATA_INDEX(p2,idx2,dtype,x);
+                if (counts) cumo_na_ndloop_wrote(lp);
                 c[nd]++;
             }
         } else {
             for (; i--;) {
+                char *q = p1;
                 CUMO_GET_DATA_STRIDE(p1,s1,dtype,x);
                 x = yield_map_with_index(x,c,a,nd,md);
                 if (i) cumo_na_ndloop_refresh_next(lp, p1, sizeof(dtype));
                 if (cumo_cuda_runtime_sync_if_busy()) { CUMO_SHOW_SYNCHRONIZE_WARNING_ONCE("<%=name%>", "<%=type_name%>"); }
+                if (own) *(dtype*)q = x;
                 CUMO_SET_DATA_STRIDE(p2,s2,dtype,x);
+                if (counts) cumo_na_ndloop_wrote(lp);
                 c[nd]++;
             }
         }

data/ext/cumo/narray/gen/tmpl/unary.c

@@ -19,6 +19,8 @@ static void
         size_t i;
         dtype x;
         <% if name == 'map' %>
+        // RObject data is host memory, which no loop stages.
+        int counts = <%= type_name == 'robject' ? '0' : 'cumo_na_ndloop_writes_existing(lp)' %>;
         if (cumo_cuda_runtime_sync_if_busy()) { CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>"); }
         <% else %>
         CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>");
@@ -30,6 +32,7 @@ static void
                     x = m_<%=name%>(x);
                     <% if name == 'map' %>if (cumo_cuda_runtime_sync_if_busy()) { CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>"); }<% end %>
                     CUMO_SET_DATA_INDEX(p2,idx2,dtype,x);
+                    <% if name == 'map' %>if (counts) cumo_na_ndloop_wrote(lp);<% end %>
                 }
             } else {
                 for (i=0; i<n; i++) {
@@ -37,6 +40,7 @@ static void
                     x = m_<%=name%>(x);
                     <% if name == 'map' %>if (cumo_cuda_runtime_sync_if_busy()) { CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>"); }<% end %>
                     CUMO_SET_DATA_STRIDE(p2,s2,dtype,x);
+                    <% if name == 'map' %>if (counts) cumo_na_ndloop_wrote(lp);<% end %>
                 }
             }
         } else {
@@ -46,6 +50,7 @@ static void
                     x = m_<%=name%>(x);
                     <% if name == 'map' %>if (cumo_cuda_runtime_sync_if_busy()) { CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>"); }<% end %>
                     CUMO_SET_DATA_INDEX(p2,idx2,dtype,x);
+                    <% if name == 'map' %>if (counts) cumo_na_ndloop_wrote(lp);<% end %>
                 }
             } else {
                 //<% if need_align %>
@@ -58,6 +63,7 @@ static void
                             x = m_<%=name%>(((dtype*)p1)[i]);
                             if (cumo_cuda_runtime_sync_if_busy()) { CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>"); }
                             ((dtype*)p2)[i] = x;
+                            if (counts) cumo_na_ndloop_wrote(lp);
                             <% else %>
                             ((dtype*)p2)[i] = m_<%=name%>(((dtype*)p1)[i]);
                             <% end %>
@@ -72,6 +78,7 @@ static void
                             x = m_<%=name%>(*(dtype*)p1);
                             if (cumo_cuda_runtime_sync_if_busy()) { CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>"); }
                             *(dtype*)p2 = x;
+                            if (counts) cumo_na_ndloop_wrote(lp);
                             <% else %>
                             *(dtype*)p2 = m_<%=name%>(*(dtype*)p1);
                             <% end %>
@@ -87,6 +94,7 @@ static void
                     x = m_<%=name%>(x);
                     <% if name == 'map' %>if (cumo_cuda_runtime_sync_if_busy()) { CUMO_SHOW_SYNCHRONIZE_FIXME_WARNING_ONCE("<%=name%>", "<%=type_name%>"); }<% end %>
                     CUMO_SET_DATA_STRIDE(p2,s2,dtype,x);
+                    <% if name == 'map' %>if (counts) cumo_na_ndloop_wrote(lp);<% end %>
                 }
                 //<% end %>
             }

data/ext/cumo/narray/narray.c

@@ -1997,6 +1997,7 @@ cumo_na_store_binary(int argc, VALUE *argv, VALUE self)
         CUMO_SHOW_SYNCHRONIZE_WARNING_ONCE("cumo_na_store_binary", "any");
         cumo_cuda_runtime_device_synchronize();
         memcpy(ptr+cumo_na_get_offset(self), RSTRING_PTR(vstr)+offset, byte_size);
+        cumo_cuda_runtime_note_device_write();
     }
 
     return SIZET2NUM(byte_size);

data/ext/cumo/narray/ndloop.c

@@ -79,8 +79,17 @@ typedef struct CUMO_NA_HOST_STAGE {
     cumo_cuda_stage_t stage;
     size_t epoch;        // the launch count when the whole range was copied
     size_t row_epoch;    // the launch count when the current row was copied
+    size_t seen_epoch;   // the launch count when the loop last looked
+    size_t row_cost;     // what copying a row costs, see CUMO_NA_STAGE_COPY_COST
+    size_t range_cost;   // what copying the whole range costs
+    size_t row_debt;     // spent on single elements since the row was copied
+    size_t range_debt;   // spent on rows and elements since the range was copied
 } cumo_na_host_stage_t;
 
+// A copy into pinned memory costs a wait before its bytes, about as long as
+// moving this many bytes takes, so a copy of n bytes costs this plus n.
+#define CUMO_NA_STAGE_COPY_COST (64u << 10)
+
 #define LARG(lp,iarg) ((lp)->user.args[iarg])
 #define LITER(lp,idim,iarg) ((lp)->xargs[iarg].iter[idim])
 #define LITER_SRC(lp,idim) ((lp)->src_iter[idim])
@@ -476,6 +485,22 @@ ndloop_alloc(cumo_na_md_loop_t *lp, cumo_ndfunc_t *nf, VALUE args,
 
 static void ndloop_unstage_host_reads(cumo_na_md_loop_t *lp);
 
+// Whether the loop wrote an array that was there before it: an input it
+// overwrites, or an output it took over from an input. A new output is
+// nothing anyone has staged.
+static int
+ndloop_writes_existing(cumo_na_md_loop_t *lp)
+{
+    int j;
+
+    for (j=0; j<lp->narg; j++) {
+        if (j < lp->nin ? (lp->xargs[j].flag & CUMO_NDL_WRITE) : lp->xargs[j].flag == CUMO_NDL_READ_WRITE) {
+            return 1;
+        }
+    }
+    return 0;
+}
+
 static VALUE
 ndloop_release(VALUE vlp)
 {
@@ -483,6 +508,13 @@ ndloop_release(VALUE vlp)
     VALUE v;
     cumo_na_md_loop_t *lp = (cumo_na_md_loop_t*)(vlp);
 
+    // A host loop, and a library call such as the scan of cumsum, writes
+    // device memory without a launch to count it, and a host loop reading a
+    // staged copy of that memory reads it again only once the count moves.
+    // Counted here so that a loop left by a raise counts too.
+    if (ndloop_writes_existing(lp)) {
+        cumo_cuda_runtime_note_device_write();
+    }
     ndloop_unstage_host_reads(lp);
     for (j=0; j < lp->narg; j++) {
         v = LARG(lp,j).value;
@@ -821,6 +853,7 @@ ndloop_set_output_narray(cumo_ndfunc_t *nf, cumo_na_md_loop_t *lp, int k,
     }
 
     j = lp->nin + k;
+    lp->xargs[j].flag = flag;
     LARG(lp,j).ndim = nd = nf->aout[k].dim;
     ndloop_set_stepidx(lp, j, v, dim_map, flag);
     if (nd > 0) {
@@ -1900,6 +1933,8 @@ static void
 ndloop_stage_row(cumo_na_md_loop_t *lp)
 {
     cumo_na_host_stage_t *hs = lp->hs;
+    // Read before the copy: a write counted while it runs leaves it behind.
+    size_t now = cumo_cuda_launch_epoch_now();
     int j;
 
     for (j=0; j<hs->narg; j++) {
@@ -1910,7 +1945,38 @@ ndloop_stage_row(cumo_na_md_loop_t *lp)
         ndloop_stage_bytes(lp, j, lo, hi, &min, &len);
         ndloop_stage_dma(lp, j, min, len);
     }
-    hs->row_epoch = cumo_cuda_launch_epoch_now();
+    hs->row_epoch = now;
+    hs->row_debt = 0;
+}
+
+// Copies the whole range of every staged input again.
+static void
+ndloop_stage_range(cumo_na_md_loop_t *lp)
+{
+    cumo_na_host_stage_t *hs = lp->hs;
+    size_t now = cumo_cuda_launch_epoch_now();
+    int j;
+
+    for (j=0; j<hs->narg; j++) {
+        if (hs->ptr[j] == NULL) continue;
+        ndloop_stage_dma(lp, j, hs->min[j], hs->len[j]);
+    }
+    hs->epoch = hs->row_epoch = now;
+    hs->row_debt = hs->range_debt = 0;
+}
+
+// The bytes a row of argument j spans, wherever the row starts. Bit rows
+// cover one word more than their bits when they start mid-word.
+static size_t
+ndloop_stage_row_len(cumo_na_md_loop_t *lp, int j)
+{
+    ssize_t lo, hi;
+
+    ndloop_stage_extent(lp, j, lp->ndim, lp->hs->ndim, 0, &lo, &hi);
+    if (rb_obj_is_kind_of(LARG(lp,j).value, cumo_cBit)) {
+        return ((size_t)(hi - lo) / CUMO_NB + 2) * sizeof(CUMO_BIT_DIGIT);
+    }
+    return (size_t)(hi - lo) + LARG(lp,j).elmsz;
 }
 
 // Reads the inputs of a host loop into pinned memory and points the
@@ -1974,16 +2040,9 @@ ndloop_stage_host_reads(cumo_na_md_loop_t *lp)
         hs->rows_only = 1;
         total = 0;
         for (j=0; j<narg; j++) {
-            ssize_t lo, hi;
             size_t len;
             if (hs->ptr[j] == NULL) continue;
-            ndloop_stage_extent(lp, j, lp->ndim, nd, 0, &lo, &hi);
-            if (rb_obj_is_kind_of(LARG(lp,j).value, cumo_cBit)) {
-                // Which words a row covers depends on where it starts.
-                len = ((size_t)(hi - lo) / CUMO_NB + 2) * sizeof(CUMO_BIT_DIGIT);
-            } else {
-                len = (size_t)(hi - lo) + LARG(lp,j).elmsz;
-            }
+            len = ndloop_stage_row_len(lp, j);
             if (len > CUMO_CUDA_STAGE_CACHE_MAX) {
                 hs->ptr[j] = NULL;
                 hs->direct++;
@@ -1997,59 +2056,121 @@ ndloop_stage_host_reads(cumo_na_md_loop_t *lp)
         if (hs->direct) cumo_cuda_runtime_device_synchronize();
         if (rows == 0) return;
         cumo_cuda_runtime_stage_alloc(&hs->stage, total);
+        hs->row_cost = CUMO_NA_STAGE_COPY_COST + total;
+        hs->seen_epoch = cumo_cuda_launch_epoch_now();
         return;
     }
     if (hs->direct) cumo_cuda_runtime_device_synchronize();
     if (total == 0) return;
     cumo_cuda_runtime_stage_alloc(&hs->stage, total);
+    hs->range_cost = CUMO_NA_STAGE_COPY_COST + total;
+    hs->row_cost = CUMO_NA_STAGE_COPY_COST;
     total = 0;
     for (j=0; j<narg; j++) {
         if (hs->ptr[j] == NULL) continue;
         hs->off[j] = total;
         LARG(lp,j).ptr = hs->stage.ptr + total - hs->min[j];
-        ndloop_stage_dma(lp, j, hs->min[j], hs->len[j]);
+        hs->row_cost += ndloop_stage_row_len(lp, j);
         total += (hs->len[j] + 7) & ~(size_t)7;
     }
-    hs->epoch = hs->row_epoch = cumo_cuda_launch_epoch_now();
+    ndloop_stage_range(lp);
+    hs->seen_epoch = hs->epoch;
 }
 
 // Before each row: a row that is only copied as reached, or one that
-// device memory was written since the copy.
+// device memory was written since the copy. Once the rows copied since a
+// write have cost as much as the whole range, and nothing was written while
+// the block ran last, the whole range is copied again instead.
 static inline void
 ndloop_stage_before_row(cumo_na_md_loop_t *lp)
 {
     cumo_na_host_stage_t *hs = lp->hs;
+    size_t now;
+    int quiet;
+
     if (hs == NULL) return;
     if (hs->direct) cumo_cuda_runtime_sync_if_busy();
-    if (hs->stage.ptr && (hs->rows_only || hs->epoch != cumo_cuda_launch_epoch_now())) {
+    if (hs->stage.ptr == NULL) return;
+    now = cumo_cuda_launch_epoch_now();
+    quiet = hs->seen_epoch == now;
+    hs->seen_epoch = now;
+    if (hs->rows_only) {
         ndloop_stage_row(lp);
+    } else if (hs->epoch != now) {
+        if (quiet && hs->range_debt >= hs->range_cost) {
+            ndloop_stage_range(lp);
+        } else {
+            ndloop_stage_row(lp);
+            hs->range_debt += hs->row_cost;
+        }
     }
 }
 
+static inline cumo_na_md_loop_t *
+ndloop_of_user(cumo_na_loop_t *user)
+{
+    return (cumo_na_md_loop_t*)((char*)user - offsetof(cumo_na_md_loop_t, user));
+}
+
 // After a yield: the block may have written the element the user function
 // reads next, which is copied again on its own so that a block writing at
 // every element costs an element, not a row, each time.
 void
 cumo_na_ndloop_refresh_next(cumo_na_loop_t *user, const void *next, size_t bytes)
 {
-    cumo_na_md_loop_t *lp = (cumo_na_md_loop_t*)((char*)user - offsetof(cumo_na_md_loop_t, user));
+    cumo_na_md_loop_t *lp = ndloop_of_user(user);
     cumo_na_host_stage_t *hs = lp->hs;
-    int j;
+    size_t now;
+    int quiet, j;
 
     if (hs == NULL) return;
     if (hs->direct) cumo_cuda_runtime_sync_if_busy();
-    if (hs->stage.ptr == NULL || hs->row_epoch == cumo_cuda_launch_epoch_now()) return;
+    if (hs->stage.ptr == NULL) return;
+    now = cumo_cuda_launch_epoch_now();
+    quiet = hs->seen_epoch == now;
+    hs->seen_epoch = now;
+    if (hs->row_epoch == now) return;
+    // Elements copied one at a time since a write have cost as much as the
+    // row, and the block did not write this time: the row is copied again at
+    // once. A block that writes at every element keeps to one.
+    if (quiet && hs->row_debt >= hs->row_cost) {
+        ndloop_stage_row(lp);
+        hs->range_debt += hs->row_cost;
+        return;
+    }
     for (j=0; j<hs->narg; j++) {
         char *base = hs->stage.ptr + hs->off[j];
         if (hs->ptr[j] == NULL) continue;
         if ((const char*)next >= base && (const char*)next + bytes <= base + hs->len[j]) {
             cumo_cuda_runtime_check_status(cumo_cuda_runtime_memcpy_to_pinned(
                 (void*)next, hs->ptr[j] + ((const char*)next - LARG(lp,j).ptr), bytes));
+            hs->row_debt += CUMO_NA_STAGE_COPY_COST + bytes;
+            hs->range_debt += CUMO_NA_STAGE_COPY_COST + bytes;
             return;
         }
     }
 }
 
+int
+cumo_na_ndloop_writes_existing(cumo_na_loop_t *user)
+{
+    return ndloop_writes_existing(ndloop_of_user(user));
+}
+
+void
+cumo_na_ndloop_wrote(cumo_na_loop_t *user)
+{
+    cumo_na_md_loop_t *lp = ndloop_of_user(user);
+    cumo_na_host_stage_t *hs = lp->hs;
+    size_t before = cumo_cuda_runtime_note_device_write();
+
+    if (hs == NULL) return;
+    // The copy holds the element, so the count it adds leaves it current.
+    if (hs->seen_epoch == before) hs->seen_epoch = before + 1;
+    if (hs->row_epoch == before) hs->row_epoch = before + 1;
+    if (hs->epoch == before) hs->epoch = before + 1;
+}
+
 static void
 ndloop_unstage_host_reads(cumo_na_md_loop_t *lp)
 {

data/lib/cumo/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Cumo
-  VERSION = "0.14.0"
+  VERSION = "0.14.1"
 
   spec = Gem.loaded_specs["cumo"] if defined?(Gem)
   REVISION = (spec.metadata["cumo_revision"]&.dup&.freeze if spec && spec.full_gem_path == File.expand_path("../..", __dir__))

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cumo
 version: !ruby/object:Gem::Version
-  version: 0.14.0
+  version: 0.14.1
 platform: ruby
 authors:
 - Naotoshi Seo
@@ -370,7 +370,7 @@ metadata:
   changelog_uri: https://github.com/sonots/cumo/blob/master/CHANGELOG.md
   bug_tracker_uri: https://github.com/sonots/cumo/issues
   rubygems_mfa_required: 'true'
-  cumo_revision: v0.13.0-11-gafd8ead9
+  cumo_revision: v0.14.1
 rdoc_options: []
 require_paths:
 - lib