carray 3.0.1 → 3.0.3

data/ext/extconf.rb

@@ -107,6 +107,17 @@ if enable_config("dev-build", false)
   $stderr.puts "carray: CARRAY_DEV_BUILD enabled (smoke surface available)"
 end
 
+# --- signed overflow wraps (`-fwrapv`)
+#
+# An integer kernel that overflows answers modulo 2^N.  C leaves signed
+# overflow undefined, and a compiler that sees several operations at once
+# rewrites them as if it could not happen -- `(x + 1) > x` becomes true --
+# which carray-jit's fused expressions, built with these flags, would show.
+if try_compile("int main(void) { return 0; }", "-fwrapv")
+  $CFLAGS += " -fwrapv"
+  $carray_build_flags << "-fwrapv"
+end
+
 # --- probe for `-fopenmp-simd` (SL.1.0, PROPOSAL_REDUCTION_SIMD_LICENSE)
 #
 # `-fopenmp-simd` enables `#pragma omp simd` (and its `reduction(+:acc)`
@@ -257,6 +268,8 @@ $INSTALLFILES << ['carray_config.h', '$(archdir)']
 # (b) Not in the closure, but written for ext authors to include
 #     explicitly.  ca_obj_face.h is the Face authoring surface: an
 #     external gem defines its own Face by including it directly.
+#     ca_for_buffer.h is whole-buffer delivery to an external library;
+#     it pulls ca_sweep_engine.h.
 #
 # A header that is neither is internal, however useful it looks in-tree.
 # ca_op_powi.h (op_powi_<type> helpers for the generated kernels) and
@@ -272,11 +285,14 @@ $INSTALLFILES << ['carray_config.h', '$(archdir)']
   ca_axis_descriptor.h
   ca_monop_dispatch.h
   ca_binop_dispatch.h
+  ca_triop_dispatch.h
   ca_bincmp_dispatch.h
   ca_moncmp_dispatch.h
   ca_kernel_iterator.h
   ca_iter_substrate.h
   ca_obj_face.h
+  ca_for_buffer.h
+  ca_sweep_engine.h
 ].each do |h|
   $INSTALLFILES << [h, '$(archdir)']
 end
@@ -324,11 +340,14 @@ end
 # compiling static sources, which is one pass with no race.
 #
 # carray_kernels.stamp is the mtime witness:
-#   - here: regenerate and touch it when mkkernel.rb is newer
+#   - here: regenerate and touch it when mkkernel.rb or ca_op_powi.h (whose
+#     helper text the generator copies) is newer
 #   - in the Rakefile (rake build_ext): the same stamp triggers need_extconf
 CARRAY_KERNELS_STAMP = "carray_kernels.stamp"
 if ( not File.exist?(CARRAY_KERNELS_STAMP) ) or
-    File.stat("mkkernel.rb").mtime > File.stat(CARRAY_KERNELS_STAMP).mtime
+    %w[mkkernel.rb ca_op_powi.h].any? { |f|
+      File.stat(f).mtime > File.stat(CARRAY_KERNELS_STAMP).mtime
+    }
   # passing "." makes the generator write the split files into ext/
   unless system("ruby mkkernel.rb .")
     raise "mkkernel.rb failed to generate split kernel files"
@@ -346,6 +365,12 @@ if Dir.glob("carray_kernels_*.c").empty?
   raise "no carray_kernels_*.c files found after mkkernel.rb; check the generator"
 end
 
+# The 5^q table the decimal-to-double reader (carray_cast.c) multiplies by.
+if ( not File.exist?("ca_pow5_table.h") ) or
+    File.stat("mk_pow5_table.rb").mtime > File.stat("ca_pow5_table.h").mtime
+  system("ruby mk_pow5_table.rb")
+end
+
 if ( not File.exist?("carray_call_cfunc.c") ) or
     ( not File.exist?("carray_call_cfunc.h") ) or
     File.stat("mk_call_cfunc.rb").mtime > File.stat("carray_call_cfunc.c").mtime

data/ext/mk_call_cfunc.rb

@@ -128,83 +128,61 @@ def indent(s, prefix = "  ")
 end
 
 # --- raw ca_call_cfunc_N ----------------------------------------------------
+#
+# The per-cell loop runs in a walker function of its own, handed to
+# ca_sweep_run: a callback that raises part way through gives back the
+# operands and scratch the engine holds, rather than leaving them behind.
+#
+# The `_r` variant (reentrant: + void *userdata at signature tail) passes a
+# trailing `void *userdata` through to every per-cell `func(...)`
+# invocation as its last argument.  Naming follows POSIX convention
+# (qsort_r / bsearch_r / strtok_r), where `_r` marks a reentrant form that
+# takes a thunk so the callback no longer depends on file-static / global
+# state.  Use when the callback needs to share state with the caller (e.g.
+# accumulators, configuration flags, library plan handles) without
+# resorting to file-static plumbing.
+def emit_raw_common(n, r:)
+  name   = r ? "ca_call_cfunc_#{n}_r" : "ca_call_cfunc_#{n}"
+  fsig   = r ? void_p_list_r(n) : void_p_list(n)
+  call   = "c->func(#{args_p(n)}#{r ? ", c->userdata" : ""})"
 
-def emit_raw(n)
-  $src.puts sig_raw(n)
-  $src.puts "{"
-  $src.puts indent(<<~END_C)
-      CArray   *cx[#{n}];
-      char     *base[#{n}];
-      ca_size_t stride[#{n}];
-      char     *owned_buf[#{n}];
-      int       attached[#{n}];
-      ca_sweep_state_t state;
-      int k_op;
-
-  END_C
-  # extract CArray* from VALUE
-  (0...n).each do |k|
-    $src.puts "  TypedData_Get_Struct(rcx#{k}, CArray, &carray_data_type, cx[#{k}]);"
-  end
-  $src.puts ""
   $src.puts <<~END_C
-      /* sweep engine: per-operand acquire (alias / xmalloc + ca_xfer_all),
-         broadcast shape check, mask OR across INPUTs, mask propagate to
-         OUTPUTs.  Lifecycle template lives in ext/ca_sweep_engine.{c,h}. */
-      state.n_ops     = #{n};
-      state.fsync     = fsync;
-      state.cx        = cx;
-      state.base      = base;
-      state.stride    = stride;
-      state.owned_buf = owned_buf;
-      state.attached  = attached;
-      state.no_mask   = 0;
-      state.src_label = "ca_call_cfunc_#{n}";
-
-      ca_sweep_acquire(&state);
-
-      /* inner loop: advance per-cell ptrs and invoke user kernel func */
-      {
-        char *p[#{n}];
-        ca_size_t k;
-        if ( state.m0 ) {
-          for ( k = 0; k < state.n_kernel; k++ ) {
-            if ( ! state.m0[k] ) {
-              for ( k_op = 0; k_op < #{n}; k_op++ ) {
-                p[k_op] = base[k_op] + k * stride[k_op];
-              }
-              func(#{args_p(n)});
-            }
-          }
-        } else {
-          for ( k = 0; k < state.n_kernel; k++ ) {
+    typedef struct {
+      ca_sweep_state_t *st;
+      void (*func)(#{fsig});
+      void *userdata;
+    } #{name}_ctx_t;
+
+    static VALUE
+    #{name}_walk (VALUE arg)
+    {
+      #{name}_ctx_t *c = (#{name}_ctx_t *) arg;
+      ca_sweep_state_t *st = c->st;
+      char *p[#{n}];
+      ca_size_t k;
+      int k_op;
+      if ( st->m0 ) {
+        for ( k = 0; k < st->n_kernel; k++ ) {
+          if ( ! st->m0[k] ) {
             for ( k_op = 0; k_op < #{n}; k_op++ ) {
-              p[k_op] = base[k_op] + k * stride[k_op];
+              p[k_op] = st->base[k_op] + k * st->stride[k_op];
             }
-            func(#{args_p(n)});
+            #{call};
+          }
+        }
+      } else {
+        for ( k = 0; k < st->n_kernel; k++ ) {
+          for ( k_op = 0; k_op < #{n}; k_op++ ) {
+            p[k_op] = st->base[k_op] + k * st->stride[k_op];
           }
+          #{call};
         }
       }
-
-      ca_sweep_release(&state);
-
-      return rcx0;
+      return Qnil;
     }
 
   END_C
-end
-
-# --- raw ca_call_cfunc_N_r (reentrant: + void *userdata at signature tail) --
-#
-# Variant of emit_raw with a trailing `void *userdata` parameter passed
-# through to every per-cell `func(...)` invocation as its last argument.
-# Naming follows POSIX convention (qsort_r / bsearch_r / strtok_r), where
-# `_r` marks a reentrant form that takes a thunk so the callback no longer
-# depends on file-static / global state.  Use when the callback needs to
-# share state with the caller (e.g. accumulators, configuration flags,
-# library plan handles) without resorting to file-static plumbing.
-def emit_raw_r(n)
-  $src.puts sig_raw_r(n)
+  $src.puts(r ? sig_raw_r(n) : sig_raw(n))
   $src.puts "{"
   $src.puts indent(<<~END_C)
       CArray   *cx[#{n}];
@@ -213,7 +191,7 @@ def emit_raw_r(n)
       char     *owned_buf[#{n}];
       int       attached[#{n}];
       ca_sweep_state_t state;
-      int k_op;
+      #{name}_ctx_t ctx;
 
   END_C
   (0...n).each do |k|
@@ -221,8 +199,9 @@ def emit_raw_r(n)
   end
   $src.puts ""
   $src.puts <<~END_C
-      /* sweep engine: same lifecycle as ca_call_cfunc_#{n}; the difference is
-         the per-cell `func(...)` call has `userdata` as its last argument. */
+      /* sweep engine: per-operand acquire (alias / xmalloc + ca_xfer_all),
+         operand pairing, mask OR across INPUTs, mask propagate to OUTPUTs.
+         Lifecycle template lives in ext/ca_sweep_engine.{c,h}. */
       state.n_ops     = #{n};
       state.fsync     = fsync;
       state.cx        = cx;
@@ -230,34 +209,14 @@ def emit_raw_r(n)
       state.stride    = stride;
       state.owned_buf = owned_buf;
       state.attached  = attached;
-      state.no_mask   = 0;
-      state.src_label = "ca_call_cfunc_#{n}_r";
+      state.src_label = "#{name}";
 
       ca_sweep_acquire(&state);
 
-      {
-        char *p[#{n}];
-        ca_size_t k;
-        if ( state.m0 ) {
-          for ( k = 0; k < state.n_kernel; k++ ) {
-            if ( ! state.m0[k] ) {
-              for ( k_op = 0; k_op < #{n}; k_op++ ) {
-                p[k_op] = base[k_op] + k * stride[k_op];
-              }
-              func(#{args_p_r(n)});
-            }
-          }
-        } else {
-          for ( k = 0; k < state.n_kernel; k++ ) {
-            for ( k_op = 0; k_op < #{n}; k_op++ ) {
-              p[k_op] = base[k_op] + k * stride[k_op];
-            }
-            func(#{args_p_r(n)});
-          }
-        }
-      }
-
-      ca_sweep_release(&state);
+      ctx.st       = &state;
+      ctx.func     = func;
+      ctx.userdata = #{r ? "userdata" : "NULL"};
+      ca_sweep_run(&state, #{name}_walk, (VALUE) &ctx);
 
       return rcx0;
     }
@@ -265,6 +224,9 @@ def emit_raw_r(n)
   END_C
 end
 
+def emit_raw(n)   ; emit_raw_common(n, r: false); end
+def emit_raw_r(n) ; emit_raw_common(n, r: true);  end
+
 # --- typed dispatchers: M outputs, N inputs ---------------------------------
 #
 # The four typed emitters (cfunc / cfunc_r / cslab / cslab_r) differ in one
@@ -418,7 +380,7 @@ def sig_slab_r(n)
   "VALUE\nca_call_cslab_#{n}_r (ca_cslab_r_t func, const char *fsync,\n                               #{value_param_list(n)},\n                               void *userdata)"
 end
 
-def emit_slab_body(n, name, call)
+def emit_slab_body(n, name, walker, userdata)
   $src.puts indent(<<~END_C)
       CArray   *cx[#{n}];
       char     *base[#{n}];
@@ -427,6 +389,7 @@ def emit_slab_body(n, name, call)
       char     *owned_buf[#{n}];
       int       attached[#{n}];
       ca_sweep_state_t state;
+      ca_cslab_ctx_t ctx;
 
   END_C
   (0...n).each do |k|
@@ -448,21 +411,15 @@ def emit_slab_body(n, name, call)
       state.stride    = stride;
       state.owned_buf = owned_buf;
       state.attached  = attached;
-      state.no_mask   = 0;
       state.src_label = "#{name}";
 
       ca_sweep_acquire_chunked(&state);
 
-      /* outer loop: hand the author one chunk at a time.  base[] is rewritten
-         per chunk by ca_sweep_next_chunk -- for a non-alias INPUT it points
-         at the arena scratch the chunk was just gathered into, which is
-         packed, so stride[] is the element size and the author's inner loop
-         sees contiguous data. */
-      while ( ca_sweep_next_chunk(&state) ) {
-        #{call};
-      }
-
-      ca_sweep_release_chunked(&state);
+      ctx.st       = &state;
+      ctx.func     = #{userdata == "NULL" ? "func" : "NULL"};
+      ctx.func_r   = #{userdata == "NULL" ? "NULL" : "func"};
+      ctx.userdata = #{userdata};
+      ca_sweep_run_chunked(&state, #{walker}, (VALUE) &ctx);
 
       return rcx0;
     }
@@ -473,8 +430,7 @@ end
 def emit_slab(n)
   $src.puts sig_slab(n)
   $src.puts "{"
-  emit_slab_body(n, "ca_call_cslab_#{n}",
-                 "func(base, stride, state.chunk_n, ca_sweep_chunk_mask(&state))")
+  emit_slab_body(n, "ca_call_cslab_#{n}", "ca_cslab_walk", "NULL")
 end
 
 # Variant of emit_slab with a trailing `void *userdata` parameter passed
@@ -483,8 +439,7 @@ end
 def emit_slab_r(n)
   $src.puts sig_slab_r(n)
   $src.puts "{"
-  emit_slab_body(n, "ca_call_cslab_#{n}_r",
-                 "func(base, stride, state.chunk_n, ca_sweep_chunk_mask(&state), userdata)")
+  emit_slab_body(n, "ca_call_cslab_#{n}_r", "ca_cslab_r_walk", "userdata")
 end
 
 # --- header declaration emitters --------------------------------------------
@@ -557,14 +512,44 @@ $src.puts <<~END_C
   #include "ca_sweep_engine.h"
   #include <string.h>
 
-  /* The chunk's iteration mask, or NULL when no INPUT operand carried one.
-     m0 is chunk-sized and re-gathered per chunk by ca_sweep_next_chunk, so
-     it is already the slice -- one byte per cell, indexed 0..chunk_n-1
-     alongside base[] and stride[]. */
-  static const boolean8_t *
-  ca_sweep_chunk_mask (ca_sweep_state_t *st)
+  /* Chunk walk for the ca_call_cslab_N family, run by ca_sweep_run_chunked
+     so that a callback raising part way through gives back what the engine
+     holds.  Hands the author one chunk at a time.  base[] is rewritten per
+     chunk by ca_sweep_next_chunk -- for a non-alias INPUT it points at the
+     arena scratch the chunk was just gathered into, which is packed, so
+     stride[] is the element size and the author's inner loop sees
+     contiguous data.  m0 is the chunk's iteration mask, or NULL when no
+     INPUT operand carried one: chunk-sized and re-gathered per chunk, so it
+     is already the slice -- one byte per cell, indexed 0..chunk_n-1
+     alongside base[] and stride[].  The arity does not appear here: the
+     operands reach the callback through base[] / stride[]. */
+  typedef struct {
+    ca_sweep_state_t *st;
+    ca_cslab_t        func;      /* ca_call_cslab_N   */
+    ca_cslab_r_t      func_r;    /* ca_call_cslab_N_r */
+    void             *userdata;
+  } ca_cslab_ctx_t;
+
+  static VALUE
+  ca_cslab_walk (VALUE arg)
   {
-    return st->m0;
+    ca_cslab_ctx_t *c = (ca_cslab_ctx_t *) arg;
+    ca_sweep_state_t *st = c->st;
+    while ( ca_sweep_next_chunk(st) ) {
+      c->func(st->base, st->stride, st->chunk_n, st->m0);
+    }
+    return Qnil;
+  }
+
+  static VALUE
+  ca_cslab_r_walk (VALUE arg)
+  {
+    ca_cslab_ctx_t *c = (ca_cslab_ctx_t *) arg;
+    ca_sweep_state_t *st = c->st;
+    while ( ca_sweep_next_chunk(st) ) {
+      c->func_r(st->base, st->stride, st->chunk_n, st->m0, c->userdata);
+    }
+    return Qnil;
   }
 
 END_C

data/ext/mk_pow5_table.rb

@@ -0,0 +1,35 @@
+# Generates ca_pow5_table.h: for each decimal exponent q in [-342, 308], a
+# 128-bit approximation of 5^q scaled by a power of two into [2^127, 2^128),
+# as the Eisel-Lemire decimal-to-binary64 conversion takes it.
+#
+# q >= 0:        floor(5^q * 2^k)            (exact while 5^q < 2^128, q <= 55)
+# -27 <= q < 0:  ceil(2^k / 5^-q)
+# q < -27:       floor(2^k / 5^-q)
+# with k chosen so the value has exactly 128 bits.
+
+Q_MIN, Q_MAX = -342, 308
+rows = (Q_MIN..Q_MAX).map do |q|
+  if q >= 0
+    v = 5**q
+    v <<= 128 - v.bit_length if v.bit_length < 128
+    v >>= v.bit_length - 128 if v.bit_length > 128
+  else
+    p5 = 5**-q
+    k = 127 + p5.bit_length
+    v, r = (1 << k).divmod(p5)
+    v += 1 if q >= -27 && r != 0
+    v >>= v.bit_length - 128 if v.bit_length > 128
+    v <<= 128 - v.bit_length if v.bit_length < 128
+  end
+  raise "bad width at #{q}" unless v.bit_length == 128
+  [v >> 64, v & ((1 << 64) - 1)]
+end
+
+File.open(File.join(__dir__, "ca_pow5_table.h"), "w") do |f|
+  f.puts "/* Generated by mk_pow5_table.rb -- do not edit. */"
+  f.puts "#define CA_POW5_Q_MIN (#{Q_MIN})"
+  f.puts "#define CA_POW5_Q_MAX #{Q_MAX}"
+  f.puts "static const uint64_t ca_pow5_table[#{rows.size * 2}] = {"
+  rows.each { |hi, lo| f.puts format("  0x%016xULL, 0x%016xULL,", hi, lo) }
+  f.puts "};"
+end