carray-jit 0.1.2 → 0.1.3

data/lib/carray/jit/c_generator.rb

@@ -126,6 +126,15 @@ class CArray
       # would.  Only the ones a kernel actually uses are emitted, and the
       # reason each exists is at emit_complex_binary.
       COMPLEX_HELPERS = {
+        "complex_finite" => <<~C,
+          /* Complex#finite? is both parts finite, which is what Ruby asks
+             and is not what isfinite of a complex would mean. */
+          static inline int
+          carray_jit_complex_finite (double _Complex z)
+          {
+            return isfinite(creal(z)) && isfinite(cimag(z));
+          }
+        C
         "complex_add_real" => <<~C,
           /* Ruby leaves the imaginary part of `z + x` exactly as it was,
              rather than adding the real operand's zero to it. */
@@ -159,9 +168,61 @@ class CArray
             return CMPLX(creal(z), y + cimag(z));
           }
         C
+        "safe_multiply" => <<~C,
+          /* complex.c's safe_mul, which every part of a Ruby complex product
+             goes through: a zero against a number that is not a zero or a
+             NaN meets only that number's sign, so a zero times an infinity is
+             a signed zero rather than C's NaN.  C's own complex `*` recovers
+             from infinities by Annex G's rules instead, which are not these. */
+          static inline double
+          carray_jit_safe_multiply (double a, double b)
+          {
+            if ( a != 0 && b == 0 && ! isnan(a) ) a = signbit(a) ? -1.0 : 1.0;
+            else if ( b != 0 && a == 0 && ! isnan(b) ) b = signbit(b) ? -1.0 : 1.0;
+            return a * b;
+          }
+        C
+        "complex_multiply" => <<~C,
+          /* Ruby's `a * b` for two Complex numbers, part by part in
+             complex.c's order: comp_mul.  safe_mul differs from a plain
+             product only for a zero against an infinity, which a plain
+             product answers NaN -- so the plain parts are the answer unless
+             one of them is a NaN, and only then is it worked out again the
+             safe way.  C's own `*` takes the same shape, and costs the same. */
+          static inline double _Complex
+          carray_jit_complex_multiply (double _Complex a, double _Complex b)
+          {
+            double are = creal(a), aim = cimag(a);
+            double bre = creal(b), bim = cimag(b);
+            double re = are * bre - aim * bim, im = are * bim + aim * bre;
+            if ( isnan(re) || isnan(im) ) {
+              re = carray_jit_safe_multiply(are, bre) - carray_jit_safe_multiply(aim, bim);
+              im = carray_jit_safe_multiply(are, bim) + carray_jit_safe_multiply(aim, bre);
+            }
+            return CMPLX(re, im);
+          }
+        C
+        "real_multiply_complex" => <<~C,
+          /* `x * z` coerces x to a Complex whose imaginary part is an exact
+             zero, and multiplies in full.  That zero meets safe_mul as any
+             zero does, so it is written as 0.0 here and gives the same parts;
+             only an addition treats an exact zero differently, and here the
+             zeros are multiplied first. */
+          static inline double _Complex
+          carray_jit_real_multiply_complex (double x, double _Complex b)
+          {
+            double bre = creal(b), bim = cimag(b);
+            double re = x * bre - 0.0 * bim, im = x * bim + 0.0 * bre;
+            if ( isnan(re) || isnan(im) ) {
+              re = carray_jit_safe_multiply(x, bre) - carray_jit_safe_multiply(0.0, bim);
+              im = carray_jit_safe_multiply(x, bim) + carray_jit_safe_multiply(0.0, bre);
+            }
+            return CMPLX(re, im);
+          }
+        C
         "complex_mul_real" => <<~C,
-          /* `z * x` scales each part.  `x * z` does not -- Ruby coerces the
-             x and multiplies in full -- so it is left to C's operator. */
+          /* `z * x` scales each part, with no safe_mul: Ruby's Complex#*
+             takes a real operand apart and multiplies each part by it. */
           static inline double _Complex
           carray_jit_complex_mul_real (double _Complex z, double x)
           {
@@ -256,8 +317,10 @@ class CArray
       BORDER_RULES = [:zero, :clamp, :wrap].freeze
 
       def initialize (analyzer, storage_types, scalar_types, c_functions: {},
+                      randoms: {},
                       masked: false, reassociate: false,
-                      steps: nil, origin: nil, block_source: nil, border: nil)
+                      steps: nil, origin: nil, block_source: nil, border: nil,
+                      scalar_parameters: [])
         @masked = masked
         # What a window read gets where it falls off the array.  Nil for every
         # kernel but a stencil's, and for a stencil whose border is the frame
@@ -282,25 +345,47 @@ class CArray
           position += analyzer.array_ranks.fetch(array, analyzer.rank)
         end
         @array_ranks = analyzer.array_ranks
-        @reals = analyzer.scalar_names.select { |name| scalar_types[name] == :double }.sort
-        @integers = analyzer.scalar_names.select { |name| scalar_types[name] == :int64 }.sort
+        # A compiled function's scalars are the parameters of its own C
+        # signature: they arrive by the ABI, in whatever type the declaration
+        # named, so no buffer carries them and they take no bus.  Set aside
+        # here, which leaves the check below about captures -- what it was
+        # always about.
+        carried_names = analyzer.scalar_names - scalar_parameters
+        @reals = carried_names.select { |name| scalar_types[name] == :double }.sort
+        @integers = carried_names.select { |name| scalar_types[name] == :int64 }.sort
         # A captured Complex rides in the reals buffer as its two parts, so
         # that the kernel signature stays the one shape every kernel has.
-        @complexes = analyzer.scalar_names.select { |name| scalar_types[name] == :complex }.sort
-        # Three buses and no fourth: a capture whose type is none of these
-        # would be packed into nothing and read as whatever the slot held, so
-        # it is caught here rather than at the cell it computes wrongly.
-        carried = @reals.size + @integers.size + @complexes.size
-        unless carried == analyzer.scalar_names.size
-          missing = analyzer.scalar_names - @reals - @integers - @complexes
+        @complexes = carried_names.select { |name| scalar_types[name] == :complex }.sort
+        # And a captured uint64 rides in the integers buffer, which is the
+        # same eight bytes read the other way: the caller packs the value's
+        # bits and the kernel casts the slot back, so nothing is lost and the
+        # signature keeps its three buffers.  Behind the signed ones, in the
+        # order the caller packs them.
+        @unsigned_integers =
+          carried_names.select { |name| scalar_types[name] == :uint64 }.sort
+        # Three buffers and no fourth: a capture whose type rides in none of
+        # them would be packed into nothing and read as whatever the slot
+        # held, so it is caught here rather than at the cell it computes
+        # wrongly.
+        carried = @reals.size + @integers.size + @complexes.size +
+                  @unsigned_integers.size
+        unless carried == carried_names.size
+          missing = carried_names - @reals - @integers - @complexes -
+                    @unsigned_integers
           raise Error,
                 "captured #{missing.join(", ")} travel in none of the kernel's " \
-                "three scalar buses; a computation type was added without a " \
+                "scalar buffers; a computation type was added without a " \
                 "way to hand a value of it to the C"
         end
         # Only the ones the block actually called, in a fixed order, because
         # the caller packs the addresses into `functions` by this order.
         @c_functions = analyzer.c_function_names.sort.to_h { |name| [name, c_functions.fetch(name)] }
+        # The generators the block drew from, by the name it drew through.
+        # Unlike a C function, one takes no slot in any buffer: its C is
+        # pasted and its state is an address array, so what is kept here is
+        # only which generator each name is -- the kind, to paste, and the
+        # symbol to call.
+        @randoms = analyzer.random_names.to_h { |name| [name, randoms.fetch(name)] }
         # A function written here is pasted into this kernel and called by its
         # symbol; a borrowed one arrives as an address.  Only the second kind
         # takes a slot in `functions`, so it is that hash the caller packs by.
@@ -330,10 +415,14 @@ class CArray
         @uses_clamp = false
         @uses_wrap = false
         @uses_real_arg = false
+        @clamp_types = []
+        @gamma_types = []
         @complex_helpers = []
         @uses_floor_divide = false
         @uses_floor_modulo = false
         @uses_integer_power = false
+        @uses_integer_abs = false
+        @uses_whole_number = false
         @uses_unsigned_divide = false
         @uses_unsigned_modulo = false
         @uses_unsigned_power = false
@@ -350,17 +439,28 @@ class CArray
         # this kernel is what answers for them.  The codes agree because they
         # are taken from the messages, not counted off.
         @raise_messages = {}
-        @pasted_functions.each_value do |function|
+        pasted_closure.each do |function|
           (function.raise_messages || {}).each do |code, message|
             register_raise(code, message)
           end
         end
+        # The intrinsic helpers the body asked for, as [name, storage, cells]
+        # -- a list used as a set, the way @clamp_types is.  A pasted body's
+        # are merged into it by #preamble.
+        @intrinsic_needs = []
+        # Whether some body pasted into this file clears an array of its own.
+        @pasted_clears_a_local_array = false
         @position_temporaries = {}.compare_by_identity
+        # The positions a write is being guarded by, by axis, while its body
+        # is emitted.
+        @guarded_axes = {}.compare_by_identity
         @origin = origin
         @block_source = block_source
+        refuse_generated_indices
       end
 
-      attr_reader :arrays, :reals, :integers, :complexes, :masked, :extent_slots,
+      attr_reader :arrays, :reals, :integers, :complexes, :unsigned_integers,
+                  :masked, :extent_slots,
                   :c_functions, :pasted_functions, :address_functions,
                   :address_arrays, :address_parameters,
                   # The messages `raise` in the block gave, by the code a cell
@@ -373,6 +473,14 @@ class CArray
         @uses_error_flag
       end
 
+      # Whether some array the body made is one of the zeroed spellings, which
+      # is what `<string.h>` is included for.  A pasted function's body is
+      # emitted into this file too, and when a function body may make one of
+      # its own this has to ask the pasted ones as well.
+      def clears_a_local_array?
+        @analyzer.clears_a_local_array? || @pasted_clears_a_local_array
+      end
+
       def generate
         strided = build_body(false)
         contiguous = build_body(true)
@@ -412,8 +520,8 @@ class CArray
         @own_symbol = name
         @in_function = true
         @error_parameter = error_parameter
+        @body_reports = false
         @contiguous = true
-        @declared_locals = {}
         @temporary_count = 0
         @carried_masks = []
         statements = @analyzer.body.statements
@@ -422,6 +530,7 @@ class CArray
         @returns_nothing = return_type.nil?
         held = @returns_nothing ? statements : statements[0..-2]
         lines = held.map { |statement| emit_statement(statement, "  ") }.join
+        lines = local_declarations(@analyzer.body, "  ") + lines
         value = @returns_nothing ? nil : emit(statements.last, return_type)
         parameters = parameter_names.zip(parameter_c_types)
                        .map { |parameter, type| type.declare(parameter) }
@@ -434,26 +543,58 @@ class CArray
         # Kept apart from the file it is compiled in: the definition alone is
         # what a kernel pastes into its own translation unit, where the
         # includes are already written and the helpers are shared.
-        @function_definition =
-          "#{return_c_type}\n#{name} (#{parameters.join(', ')})\n{\n" +
-          lines + (@returns_nothing ? "}\n" : "  return #{value};\n}\n")
-        preamble + flag + @function_definition
+        opening = "#{return_c_type}\n#{name} (#{parameters.join(', ')})\n{\n"
+        body = lines + (@returns_nothing ? "}\n" : "  return #{value};\n}\n")
+        @function_definition = opening + body
+        preamble + flag + opening + standing_gate + body
       end
 
       attr_reader :function_definition
 
+      # A body standing on its own does no more work once its flag stands.
+      #
+      # The `!ERROR_FLAG` at each `raise` keeps a later report from overwriting
+      # an earlier one, and that is enough for a caller that stops: a kernel
+      # ends its loop, and `CFunction#call` was one call to begin with.  A
+      # library handed the address does not stop.  It calls again, and without
+      # this the body skips the report it has already made and answers as
+      # though nothing had happened -- so an adaptive routine converges on
+      # values that are fiction, which is worse than a wrong answer because it
+      # looks like a right one.
+      #
+      # What the gate says is that the body does no more work.  It does not
+      # say what the call is worth: the flag is the contract, and the zero is
+      # a courtesy to a caller who never looks at it.  A `void` body has no
+      # courtesy to offer and leaves its out-parameters alone, which is the
+      # same refusal to work.
+      #
+      # It goes in the file and not in the definition, so that the form a
+      # kernel pastes cannot pick it up whichever way the two forms are
+      # generated.  A kernel reads its slot once, after a loop it stopped
+      # itself, and has no use for it.
+      def standing_gate
+        return "" unless @uses_error_flag && !@error_parameter
+        "  if ( #{ERROR_FLAG} ) #{@returns_nothing ? "return;" : "return 0;"}\n\n"
+      end
+
       # What the body took from the preamble, so that a definition pasted
       # somewhere else can be given the same helpers.  Only the ones a pasted
       # function can want are here: the rest -- the index check, the two
       # flooring helpers -- report through the error flag, and a function that
       # touches the flag is not pasted at all.
       def helper_needs
-        { :integer_power => @uses_integer_power,
+        { :intrinsics => @intrinsic_needs.dup,
+          :clears_a_local_array => clears_a_local_array?,
+          :integer_power => @uses_integer_power,
+          :integer_abs => @uses_integer_abs,
+          :whole_number => @uses_whole_number,
           :unsigned_divide => @uses_unsigned_divide,
           :unsigned_modulo => @uses_unsigned_modulo,
           :unsigned_power => @uses_unsigned_power,
           :floor_modulo_float => @uses_floor_modulo_float,
           :real_arg => @uses_real_arg,
+          :clamp => @clamp_types.dup,
+          :gamma => @gamma_types.dup,
           :complex => @complex_helpers.dup,
           :index_check => @uses_index_check,
           :floor_divide => @uses_floor_divide,
@@ -547,27 +688,64 @@ class CArray
         # A pasted body wants the same helpers here that it had in its own
         # file, and it is emitted below them, so this is asked before any of
         # them is written out.
-        @pasted_functions.each_value do |function|
+        pasted_closure.each do |function|
           needs = function.helpers || {}
           @uses_integer_power ||= needs[:integer_power]
+          @uses_integer_abs ||= needs[:integer_abs]
+          @uses_whole_number ||= needs[:whole_number]
           @uses_unsigned_divide ||= needs[:unsigned_divide]
           @uses_unsigned_modulo ||= needs[:unsigned_modulo]
           @uses_unsigned_power ||= needs[:unsigned_power]
           @uses_floor_modulo_float ||= needs[:floor_modulo_float]
           @uses_real_arg ||= needs[:real_arg]
+          @clamp_types |= needs[:clamp] || []
+          @gamma_types |= needs[:gamma] || []
           @complex_helpers |= needs[:complex] || []
           @uses_index_check ||= needs[:index_check]
           @uses_floor_divide ||= needs[:floor_divide]
           @uses_floor_modulo ||= needs[:floor_modulo]
-        end
-        text = +"#include <stdint.h>\n#include <math.h>\n#include <complex.h>\n" \
-                "#include <stdio.h>\n\n"
+          # A union rather than an or: two bodies may want a `sort` of one
+          # type and length and a `sum` of another, and the preamble owes one
+          # helper for each.  The same shape `:clamp` and `:gamma` take.
+          @intrinsic_needs |= needs[:intrinsics] || []
+          # And whether anything anywhere in this file clears an array of its
+          # own, which is what `<string.h>` is included for.  A pasted body's
+          # `memset` is in this translation unit, so it is this file's header.
+          @pasted_clears_a_local_array ||= !!needs[:clears_a_local_array]
+        end
+        # `stddef.h` is here for the signature's sake rather than the body's:
+        # a declaration may be written with `size_t` or `ptrdiff_t`, and that
+        # is the header those come from.  `stdio.h` happens to declare
+        # `size_t` as well, which is why the omission went unnoticed until
+        # `ptrdiff_t` was written down.
+        text = +"#include <stdint.h>\n#include <stddef.h>\n" \
+                "#include <math.h>\n#include <complex.h>\n" \
+                "#include <stdio.h>\n"
+        # `memset`, and only where a body clears an array of its own.  A
+        # kernel that clears nothing generates what it always did.
+        text << "#include <string.h>\n" if clears_a_local_array?
+        # malloc and free, for the arrays this kernel allocates.  libc, the
+        # same ground `memset` above stands on: it needs no GVL and touches
+        # no Ruby value, which is what kept `ALLOCV` out of here.
+        text << "#include <stdlib.h>\n" unless heap_arrays.empty?
+        text << "\n"
         unless @address_functions.empty?
-          text << "/* The C functions the block called.  They arrive as\n" \
-                  "   addresses rather than by linkage, so nothing here says\n" \
-                  "   which library they came from. */\n"
-          @address_functions.each_key do |name|
-            text << @address_functions.fetch(name).c_declaration(c_function_type_name(name)) << "\n"
+          if @in_function
+            # A body standing on its own calls a borrowed function by its
+            # name, so what it needs is that function's own declaration --
+            # the one its author wrote -- and a linker.
+            text << "/* Declared where they are called from, and resolved " \
+                    "by the linker\n   or the loader. */\n"
+            @address_functions.each_value do |function|
+              text << "#{function.prototype};\n"
+            end
+          else
+            text << "/* The C functions the block called.  They arrive as\n" \
+                    "   addresses rather than by linkage, so nothing here says\n" \
+                    "   which library they came from. */\n"
+            @address_functions.each_key do |name|
+              text << @address_functions.fetch(name).c_declaration(c_function_type_name(name)) << "\n"
+            end
           end
           text << "\n"
         end
@@ -589,8 +767,12 @@ class CArray
                computing wide and narrowing afterwards stops agreeing with
                computing narrow. */
             static inline float
-            carray_jit_floor_modulo_float (float numerator, float denominator)
+            carray_jit_floor_modulo_float (float numerator, float denominator, int32_t *error)
             {
+              if ( denominator == 0 ) {
+                if ( error ) *error = 1;
+                return 0;
+              }
               float remainder = fmodf(numerator, denominator);
               if ( remainder != 0 ) {
                 if ( (remainder < 0) != (denominator < 0) ) remainder += denominator;
@@ -677,6 +859,54 @@ class CArray
 
           C
         end
+        if @uses_whole_number
+          text << <<~C
+            /* `floor`, `ceil`, `round` and `truncate` on a Float answer an
+               Integer in Ruby, and a NaN or an infinity has none to answer:
+               Ruby raises FloatDomainError.  A number past int64 has one, but
+               not in an int64_t, where the cast is undefined -- CArray raises
+               RangeError storing such an Integer, and so does this.  The real
+               one is for a value that stays a number: 1e20.floor into a
+               float64 cell is 1e20 in Ruby, and does not pass through int64. */
+            static inline double
+            carray_jit_whole_real (double value, int32_t *error)
+            {
+              if ( isnan(value) ) {
+                if ( error ) *error = #{FLOAT_NAN_CODE};
+              }
+              else if ( isinf(value) ) {
+                if ( error ) *error = value > 0 ? #{FLOAT_INFINITY_CODE} : #{FLOAT_NEGATIVE_INFINITY_CODE};
+              }
+              return value;
+            }
+
+            static inline int64_t
+            carray_jit_whole_int64 (double value, int32_t *error)
+            {
+              if ( ! ( value >= -9223372036854775808.0 && value < 9223372036854775808.0 ) ) {
+                carray_jit_whole_real(value, error);
+                if ( error && ! isnan(value) && ! isinf(value) ) *error = #{INTEGER_RANGE_CODE};
+                return 0;
+              }
+              return (int64_t) value;
+            }
+
+          C
+        end
+        if @uses_integer_abs
+          text << <<~C
+            /* An integer's magnitude, without llabs: that is <stdlib.h>'s,
+               which a kernel includes only when it allocates, and it leaves
+               INT64_MIN undefined.  Negated through uint64_t, INT64_MIN
+               wraps to itself -- the wrap an int64 cell has everywhere else. */
+            static inline int64_t
+            carray_jit_integer_abs (int64_t value)
+            {
+              return value < 0 ? (int64_t) (0 - (uint64_t) value) : value;
+            }
+
+          C
+        end
         COMPLEX_HELPERS.each do |name, definition|
           [false, true].each do |narrow|
             next unless @complex_helpers.include?([name, narrow])
@@ -684,6 +914,76 @@ class CArray
                             : definition) << "\n"
           end
         end
+        @gamma_types.each do |type|
+          suffix, c_type, gamma, modf = GAMMA_C_TYPES.fetch(type)
+          table = (1..GAMMA_TABLE_LIMIT).map { |whole|
+            format_float(Math.gamma(whole.to_f), type)
+          }
+          text << <<~C
+            /* `Math.gamma` is tgamma with Ruby's two answers around it: a
+               whole number up to #{GAMMA_TABLE_LIMIT} is answered from the table Ruby
+               answers it from, and a negative whole number -- where tgamma
+               gives a NaN -- is the domain error Ruby raises.  The values
+               are the ones the Ruby that compiled this holds. */
+            static const #{c_type} carray_jit_gamma_table_#{suffix}[#{GAMMA_TABLE_LIMIT}] = {
+              #{table.each_slice(4).map { |row| row.join(", ") }.join(",\n  ")}
+            };
+
+            static inline #{c_type}
+            carray_jit_gamma_#{suffix} (#{c_type} x, int32_t *error)
+            {
+              if ( isinf(x) ) {
+                if ( signbit(x) ) {
+                  if ( error ) *error = 5;
+                  return 0;
+                }
+                return x;
+              }
+              #{c_type} whole;
+              if ( #{modf}(x, &whole) == 0 ) {
+                if ( whole < 0 ) {
+                  if ( error ) *error = 5;
+                  return 0;
+                }
+                if ( 0 < whole && whole <= #{GAMMA_TABLE_LIMIT} ) {
+                  return carray_jit_gamma_table_#{suffix}[(int)whole - 1];
+                }
+              }
+              return #{gamma}(x);
+            }
+
+          C
+        end
+        text << intrinsic_helpers
+        @clamp_types.each do |type|
+          suffix, c_type, has_nan = CLAMP_C_TYPES.fetch(type)
+          refusal = lambda { |test, code|
+            "  if ( #{test} ) {\n" \
+            "    if ( error ) *error = #{code};\n" \
+            "    return value;\n" \
+            "  }\n"
+          }
+          tests = +""
+          tests << refusal.call("isnan(low) || isnan(high)", 4) if has_nan
+          tests << refusal.call("low > high", 3)
+          tests << refusal.call("isnan(value)", 4) if has_nan
+          text << <<~C
+            /* `x.clamp(low, high)`, and the two things Ruby raises
+               ArgumentError for: bounds the wrong way round, and a
+               comparison that cannot be made because a NaN is in it.  The
+               order is Ruby's -- the bounds are ordered against each other
+               before the value is looked at. */
+            static inline #{c_type}
+            carray_jit_clamp_#{suffix} (#{c_type} value, #{c_type} low, #{c_type} high, int32_t *error)
+            {
+            #{tests.chomp}
+              if ( value < low ) return low;
+              if ( value > high ) return high;
+              return value;
+            }
+
+          C
+        end
         if @uses_real_arg
           text << <<~C
             /* The argument of a real number, which Ruby reads off the sign
@@ -757,7 +1057,12 @@ class CArray
                non-zero and disagrees with it in sign -- and, for floats,
                give a zero remainder the divisor's sign, so the rule holds
                without exception.  This mirrors CArray's own `:mod` kernel in
-               `ext/mkkernel.rb`. */
+               `ext/mkkernel.rb`.
+
+               A zero divisor is Ruby's rather than CArray's: `3.0 % 0.0` raises
+               ZeroDivisionError in Ruby, a float divisor as much as an integer
+               one, where fmod and CArray answer NaN.  `3.0 / 0.0` is not that
+               case -- an infinity in Ruby as in C -- and is left alone. */
             static inline int64_t
             carray_jit_floor_modulo (int64_t numerator, int64_t denominator, int32_t *error)
             {
@@ -773,8 +1078,12 @@ class CArray
             }
 
             static inline double
-            carray_jit_floor_modulo_real (double numerator, double denominator)
+            carray_jit_floor_modulo_real (double numerator, double denominator, int32_t *error)
             {
+              if ( denominator == 0 ) {
+                if ( error ) *error = 1;
+                return 0;
+              }
               double remainder = fmod(numerator, denominator);
               if ( remainder != 0 ) {
                 if ( (remainder < 0) != (denominator < 0) ) remainder += denominator;
@@ -814,9 +1123,40 @@ class CArray
 
           C
         end
+        text << random_source
         text << pasted_definitions
       end
 
+      # The generators the block drew from, as CArray wrote them.
+      #
+      # This is the COMPLEX_HELPERS path and not the pasted-body one: what
+      # CArray hands over is a run of `static inline` functions, not one
+      # definition to be given a `static` and a name.  It goes in verbatim --
+      # reading it here to check it would be this gem deciding what CArray's
+      # generator is, which is the drift the shared text exists to prevent.
+      #
+      # Only when a draw was written, and once however many generators drew:
+      # two CArray::Rng of the same kind are the same C over different
+      # state.
+      def random_source
+        kinds = @analyzer.random_names.map { |name|
+          @randoms.fetch(name).generator
+        }.uniq.sort
+        return "" if kinds.empty?
+        # What every generator's text wants beside it and none of them owns.
+        # Once, ahead of them: two generators in this file would otherwise
+        # define it twice.
+        text = +"/* What a draw needs beside a generator, from\n" \
+                "   CArray::Rng::COMMON_SOURCE. */\n" +
+               CArray::Rng::COMMON_SOURCE + "\n"
+        text + kinds.map { |kind|
+          "/* #{kind}, from CArray::Rng::SOURCE -- the same text CArray\n" \
+          "   compiled, so a kernel continues the sequence `random!` left\n" \
+          "   off at rather than agreeing with it by construction. */\n" +
+          CArray::Rng::SOURCE.fetch(kind) + "\n"
+        }.join
+      end
+
       # The bodies of the functions written with `jit_function`, put in this
       # translation unit as statics and called by name.  An address the
       # compiler cannot see through is a call it cannot inline, and the loop
@@ -829,18 +1169,49 @@ class CArray
       # one.  The name carries the digest of the body, so two of them are the
       # same function, and one is pasted once however many names the block
       # reached it by.
+      # Every function whose definition ends up in this file: the ones the
+      # block named, and the ones those call.  Helpers and raise messages
+      # are asked of all of them, since a body pasted three deep reports
+      # through the same slot and wants the same preamble.
+      def pasted_closure
+        found = {}
+        walk = lambda do |function|
+          # A borrowed function has no definition to paste; it is declared.
+          next unless function.pasted?
+          next if found[function.symbol]
+          found[function.symbol] = function
+          (function.dependencies || []).each { |called| walk.call(called) }
+        end
+        @pasted_functions.each_value { |function| walk.call(function) }
+        found.values
+      end
+
+      # A pasted body may itself call a function compiled here, which it
+      # reaches by symbol -- so that one is pasted too, and ahead of it, or
+      # the call would be to a name this file has not defined yet.  The same
+      # `seen` covers both: one definition per symbol however many ways it
+      # was reached.
       def pasted_definitions
         seen = {}
         text = +""
         @pasted_functions.each_value do |function|
-          next if seen[function.name]
-          seen[function.name] = true
-          text << "/* #{function} */\n" \
-                  "static #{function.definition}\n"
+          text << paste_definition(function, seen)
         end
         text
       end
 
+      def paste_definition (function, seen)
+        return "" unless function.pasted?
+        return "" if seen[function.symbol]
+        seen[function.symbol] = true
+        text = +""
+        (function.dependencies || []).each do |called|
+          text << paste_definition(called, seen)
+        end
+        text << "/* #{function} */\n" \
+                "static #{function.definition}\n"
+      end
+
       # One compiled object holds both loops and picks between them once,
       # outside the loop.  The contiguous form indexes a typed pointer along
       # the innermost axis, which the compiler can vectorise; the strided form
@@ -848,7 +1219,7 @@ class CArray
       def dispatcher
         tests = @arrays.map { |array|
           "strides[#{@stride_offsets.fetch(array) + array_rank(array) - 1}] == " \
-          "(int64_t) sizeof(#{storage_c_type(array)})"
+          "(int64_t) sizeof(#{storage_c_type(array_storage(array))})"
         }
         # A kernel reaching no array cell has nothing to be contiguous about
         # -- it can only be one whose work is a call, the arrays it touches
@@ -878,7 +1249,6 @@ class CArray
 
       def build_body (contiguous)
         @contiguous = contiguous
-        @declared_locals = {}
         @temporary_count = 0
         @carried_masks = []
         # Whether this body has a way to report -- asked of the statements
@@ -886,10 +1256,202 @@ class CArray
         # kernel carries every extent of its arrays so the border rule has
         # them, which says nothing about whether a cell can fail.
         @body_reports = false
+        indent = "  " + "  " * @rank
         statements = @analyzer.body.statements.map { |statement|
-          emit_statement(statement, "  " + "  " * @rank)
+          emit_statement(statement, indent)
+        }.join
+        "{\n" + declarations + heap_prologue + loop_open +
+          local_declarations(@analyzer.body, indent) + statements +
+          loop_close + heap_epilogue + "}\n"
+      end
+
+      # The arrays this kernel allocates, in the order the block declared
+      # them, as [C name, storage, shape].
+      #
+      # Gathered from every scope rather than from the one being emitted: the
+      # allocation is at the function's entry, outside the cell loop, so a
+      # pointer declared for an array the block made inside an inner loop
+      # still stands at the top of the function.  Two sibling loops that each
+      # write `w` are two bindings and so two pointers, which is what keeps
+      # them the two arrays Ruby says they are.
+      def heap_arrays
+        @heap_arrays ||= begin
+          found = []
+          walk = lambda do |node|
+            if node.respond_to?(:array_declarations) && node.array_declarations
+              node.array_declarations.each do |name, binding, storage, shape, heap|
+                next unless heap
+                entry = [local_c_name(name, binding), storage, shape]
+                found << entry unless found.include?(entry)
+              end
+            end
+            node.children.each { |child| walk.call(child) if child.is_a?(Node) }
+          end
+          walk.call(@analyzer.body)
+          found
+        end
+      end
+
+      # One allocation per array at the entry, before the loop: the cells,
+      # and the shadow beside them where the kernel carries masks.
+      #
+      # The lengths come first and stand in `const int64_t` of their own,
+      # because each is read three times over -- to check it, to size the
+      # allocation, and at every subscript on that axis -- and the expression
+      # behind it may be anything the block wrote.  A length that is not a
+      # whole number of cells above zero is reported rather than allocated,
+      # which is what the same shape written as a number is refused for as
+      # the block is read.  So is a length whose cells would not fit in a
+      # `size_t`: the multiplication into `malloc` would wrap, and a small
+      # allocation answered for a huge one is the worst of the outcomes.
+      #
+      # That is asked of the lengths before they are multiplied, one axis at
+      # a time against what the axes before it leave of the limit.  Asked of
+      # the product, it came too late: two lengths of 2**32 multiply to a
+      # number that has already wrapped to zero, which fits, and every
+      # subscript was then checked against its own axis and let through
+      # into an allocation of nothing.  The limit is `int64_t` as well as
+      # `size_t`, since that is what the cells are counted in.
+      def heap_prologue
+        return "" if heap_arrays.empty?
+        lines = +""
+        heap_arrays.each do |variable, storage, shape|
+          shape.each_with_index do |extent, axis|
+            next if extent.is_a?(Integer)
+            lines << "  const int64_t #{heap_extent_name(variable, axis)} = " \
+                     "#{emit(extent.node, :int64)};\n"
+          end
+          room = "(uint64_t) (SIZE_MAX / sizeof(#{storage_c_type(storage)}))"
+          lines << "  const uint64_t #{heap_limit_name(variable)} =\n" \
+                   "    #{room} < (uint64_t) INT64_MAX ? #{room} : (uint64_t) INT64_MAX;\n"
+          lengths = heap_extent_texts(variable, shape)
+          tests = shape.each_with_index.filter_map { |extent, axis|
+            next if extent.is_a?(Integer)
+            "#{heap_extent_name(variable, axis)} >= 1"
+          } + lengths.each_index.map { |axis|
+            left = [heap_limit_name(variable)] +
+                   lengths.first(axis).map { |length| "(uint64_t) #{length}" }
+            "(uint64_t) #{lengths[axis]} <= #{left.join(' / ')}"
+          }
+          lines << "  const int #{heap_fits_name(variable)} =\n" \
+                   "    #{tests.join("\n    && ")};\n"
+          next if literal_shape?(shape)
+          lines << "  const int64_t #{heap_cells_name(variable)} = " \
+                   "#{heap_fits_name(variable)} ? #{lengths.join(' * ')} : 0;\n"
+        end
+        heap_arrays.each do |variable, storage, _shape|
+          lines << "  #{storage_c_type(storage)} *#{variable} = NULL;\n"
+          lines << "  uint8_t *#{local_mask_name(variable)} = NULL;\n" if @masked
+        end
+        unfit = heap_arrays.map { |variable, _storage, _shape|
+          "!#{heap_fits_name(variable)}"
+        }
+        lines << "  if ( #{unfit.join(' || ')} ) {\n" \
+                 "    if ( error ) *error = #{SHAPE_CODE};\n" \
+                 "    return;\n" \
+                 "  }\n"
+        heap_arrays.each do |variable, storage, shape|
+          cells = local_array_cells_text(variable, shape)
+          lines << "  #{variable} = malloc(sizeof(#{storage_c_type(storage)}) " \
+                   "* (size_t) #{cells});\n"
+          next unless @masked
+          lines << "  #{local_mask_name(variable)} = malloc((size_t) #{cells});\n"
+        end
+        held = heap_arrays.flat_map { |variable, _storage, _shape|
+          [variable] + (@masked ? [local_mask_name(variable)] : [])
+        }
+        lines << "  if ( #{held.map { |name| "!#{name}" }.join(' || ')} ) {\n"
+        held.each { |name| lines << "    free(#{name});\n" }
+        lines << "    if ( error ) *error = #{MEMORY_CODE};\n" \
+                 "    return;\n" \
+                 "  }\n\n"
+        lines
+      end
+
+      # Every way out of a kernel that allocated: the fall-through at the end
+      # of the loop, and the one `return` a `raise` emits, which stands
+      # inside it.  Counted rather than assumed -- a leak here is a leak per
+      # call, and the sweep calls once per chunk.
+      def heap_epilogue
+        heap_release("  ")
+      end
+
+      def heap_release (indent)
+        return "" if heap_arrays.empty?
+        heap_arrays.map { |variable, _storage, _shape|
+          "#{indent}free(#{variable});\n" +
+            (@masked ? "#{indent}free(#{local_mask_name(variable)});\n" : "")
+        }.join
+      end
+
+      def heap_extent_name (variable, axis)
+        "#{variable}__extent#{axis}"
+      end
+
+      def heap_cells_name (variable)
+        "#{variable}__cells"
+      end
+
+      def heap_limit_name (variable)
+        "#{variable}__limit"
+      end
+
+      def heap_fits_name (variable)
+        "#{variable}__fits"
+      end
+
+      def heap_extent_texts (variable, shape)
+        shape.each_with_index.map { |extent, axis|
+          extent.is_a?(Integer) ? extent.to_s : heap_extent_name(variable, axis)
+        }
+      end
+
+      def literal_shape? (shape)
+        shape.all? { |extent| extent.is_a?(Integer) }
+      end
+
+      # The locals a scope holds, declared at the head of its block with no
+      # value: every assignment below is a plain one, wherever it stands --
+      # inside a `while` or an arm of an `if` included -- and nothing is read
+      # before it is written, which the typing has already made sure of.
+      def local_declarations (scope, indent)
+        scope.declarations.map { |name, binding, type|
+          variable = local_c_name(name, binding)
+          "#{indent}#{COMPUTATION_C_TYPES.fetch(type)} #{variable};\n" +
+            (@masked ? "#{indent}uint8_t #{local_mask_name(variable)};\n" : "")
+        }.join + local_array_declarations(scope, indent)
+      end
+
+      # The arrays a scope holds, declared at the head of its block beside the
+      # locals.  The length is the one the block wrote, so the C array is a
+      # real one -- a fixed-size automatic object, not a pointer into
+      # anything -- and its cells are addressed by constants.
+      #
+      # What clears the zeroed ones is emitted where the statement stands
+      # rather than here: in Ruby a fresh array is made at that line each time
+      # it runs, so a loop's body starts from cleared cells on every pass.
+      def local_array_declarations (scope, indent)
+        scope.array_declarations.map { |name, binding, storage, shape, heap|
+          # One the kernel allocates is a pointer standing at the top of the
+          # function, declared and freed with the allocation (see
+          # #heap_prologue), so the scope it belongs to declares nothing.
+          next "" if heap
+          # One flat run of cells, whatever the rank: the subscripts are
+          # folded to a row-major offset with the strides baked in (see
+          # #local_array_reference), and a flat array is also what a C
+          # function's pointer parameter is handed.  `double m[3][4]` would
+          # be the same bytes and a second spelling to keep in step.
+          variable = local_c_name(name, binding)
+          cells = shape.inject(1, :*)
+          # Where the kernel carries masks, a shadow of one byte a cell
+          # beside the cells -- the same arrangement a scalar local has, one
+          # byte beside its value, spread over the cells.  A cell of a local
+          # array then reads like a cell of a captured one: the value, and
+          # whether it is there.
+          "#{indent}#{storage_c_type(storage)} #{variable}[#{cells}];\n" +
+            (@masked ?
+               "#{indent}uint8_t #{local_mask_name(variable)}[#{cells}];\n" : "")
         }.join
-        "{\n" + declarations + loop_open + statements + loop_close + "}\n"
       end
 
       def array_rank (array)
@@ -930,30 +1492,38 @@ class CArray
           end
         end
         @reals.each_with_index do |name, index|
-          lines << "  const double #{c_name(name)} = reals[#{index}];\n"
+          lines << "  const double #{bare_name(name)} = reals[#{index}];\n"
         end
         @complexes.each_with_index do |name, index|
           slot = @reals.size + 2 * index
-          lines << "  const double _Complex #{c_name(name)} = " \
+          lines << "  const double _Complex #{bare_name(name)} = " \
                    "CMPLX(reals[#{slot}], reals[#{slot + 1}]);\n"
         end
         @integers.each_with_index do |name, index|
-          lines << "  const int64_t #{c_name(name)} = integers[#{index}];\n"
+          lines << "  const int64_t #{bare_name(name)} = integers[#{index}];\n"
+        end
+        # The cast is the whole of how a uint64 capture travels: the caller
+        # packed the value's bits into the slot, and reading them as what they
+        # are is what puts the value back.
+        @unsigned_integers.each_with_index do |name, index|
+          lines << "  const uint64_t #{bare_name(name)} = " \
+                   "(uint64_t) integers[#{@integers.size + index}];\n"
         end
         @extent_slots.each_with_index do |(array, axis), slot|
           lines << "  const int64_t #{extent_name(array, axis)} = " \
-                   "integers[#{@integers.size + slot}];\n"
+                   "integers[#{@integers.size + @unsigned_integers.size + slot}];\n"
         end
         @address_functions.each_key.with_index do |name, index|
-          lines << "  const #{c_function_type_name(name)} #{c_name(name)} = " \
+          lines << "  const #{c_function_type_name(name)} #{bare_name(name)} = " \
                    "(#{c_function_type_name(name)}) functions[#{index}];\n"
         end
         @address_arrays.each_with_index do |array, index|
           # Typed by the array rather than by the parameter it will be passed
           # to: the caller has already checked that the two agree, and the
           # array is the one that owns the memory.
-          lines << "  #{storage_c_type(array)} *const #{c_name(array)} = " \
-                   "(#{storage_c_type(array)} *) data[#{index}];\n"
+          storage = storage_c_type(array_storage(array))
+          lines << "  #{storage} *const #{bare_name(array)} = " \
+                   "(#{storage} *) data[#{index}];\n"
         end
         lines.empty? ? "" : lines.join + "\n"
       end
@@ -997,6 +1567,20 @@ class CArray
       def emit_statement (statement, indent)
         case statement
         when Assignment    then emit_assignment(statement, indent)
+        when ParallelAssignment then
+          # The values, and then the writes.  They are ordinary statements by
+          # the time they reach here -- the analyzer made them so -- and the
+          # order they are in is the whole of what a parallel assignment
+          # means.
+          statement.statements.map { |inner|
+            emit_statement(inner, indent)
+          }.join
+        when IntrinsicStatement then emit_intrinsic_statement(statement, indent)
+        when LocalArrayDeclaration then emit_local_array_clearing(statement, indent)
+        when LocalArrayWrite then guarded(statement, indent) { |inner|
+                                  emit_local_array_write(statement, inner) }
+        when LocalArrayMaskWrite then guarded(statement, indent) { |inner|
+                                  emit_local_array_mask_write(statement, inner) }
         when ElementWrite  then guarded(statement, indent) { |inner|
                                   emit_element_write(statement, inner) }
         when MaskWrite     then guarded(statement, indent) { |inner|
@@ -1019,6 +1603,33 @@ class CArray
 
       # 1 and 2 are the divisor that was not there and the subscript that ran
       # off its array.  A `raise` in the block takes a code from here up.
+      # What a kernel says when it could not start: a shape it worked out
+      # that is no shape, and an allocation that failed.  Negative, because
+      # the positive codes from RAISE_CODE_FLOOR up are a digest of a
+      # `raise` message and a fixed one among them could collide; nothing
+      # reaches for a negative code, so the fixed ones are kept down here,
+      # the kernel's own.
+      SHAPE_CODE = -1
+      MEMORY_CODE = -2
+
+      # And what a rounding reports where Ruby has no Integer to answer, or
+      # has one an int64 cannot hold.  Negative for the same reason: fixed
+      # codes, which a `raise` digest must not land on.
+      FLOAT_NAN_CODE = -3
+      FLOAT_INFINITY_CODE = -4
+      FLOAT_NEGATIVE_INFINITY_CODE = -5
+      INTEGER_RANGE_CODE = -6
+
+      # The fixed failures a body reports, as the exception each one raises.
+      # A kernel and a compiled function both look here, so the two say the
+      # same thing for the same cell.
+      FIXED_FAILURES = {
+        FLOAT_NAN_CODE => [FloatDomainError, "NaN"],
+        FLOAT_INFINITY_CODE => [FloatDomainError, "Infinity"],
+        FLOAT_NEGATIVE_INFINITY_CODE => [FloatDomainError, "-Infinity"],
+        INTEGER_RANGE_CODE => [RangeError, "a rounded Float is past the range of int64"],
+      }.freeze
+
       RAISE_CODE_FLOOR = 3
 
       # The code is taken from the message rather than counted off as messages
@@ -1070,7 +1681,8 @@ class CArray
         # The value is not the answer; the flag says so.
         leaving = @in_function && !@returns_nothing ? "return 0;" : "return;"
         "#{indent}if ( #{tests.join(' && ')} ) {\n" \
-        "#{indent}  #{report}\n" \
+        "#{indent}  #{report}\n" +
+        heap_release("#{indent}  ") +
         "#{indent}  #{leaving}\n" \
         "#{indent}}\n"
       end
@@ -1229,17 +1841,28 @@ class CArray
       # report -- but writing outside it cannot: the report would arrive after
       # the damage.  So a scatter works its positions out first, and writes
       # only if every one of them is inside.
+      #
+      # Each position carries the text of the extent it is tested against
+      # rather than an array and an axis to look one up from.  An operand's
+      # extent is a variable the kernel was handed, and there are extents that
+      # are not one of those; what the test needs is the text either way.
       def guarded (write, indent)
         positions = scattered_positions(write)
         return yield(indent) if positions.empty?
 
         lines = "#{indent}{\n"
-        positions.each do |temporary, node, array, axis|
+        # Held by axis as well as by node: an axis whose length the kernel
+        # works out is tested here even where the position is an index, and
+        # an index is not a node the reference can look itself up by.
+        held = {}
+        positions.each_with_index do |(temporary, node, _extent), axis|
           lines << "#{indent}  const int64_t #{temporary} = #{emit(node, :int64)};\n"
           @position_temporaries[node] = temporary
+          held[positions_axis(write, axis)] = temporary
         end
-        test = positions.map { |temporary, _, array, axis|
-          "#{temporary} >= 0 && #{temporary} < #{extent_name(array, axis)}"
+        @guarded_axes[write] = held
+        test = positions.map { |temporary, _node, extent|
+          "#{temporary} >= 0 && #{temporary} < #{extent}"
         }.join(" && ")
         lines << "#{indent}  if ( #{test} ) {\n"
         lines << yield("#{indent}    ")
@@ -1247,19 +1870,74 @@ class CArray
         lines << "#{indent}    if ( #{error_argument} ) *#{error_argument} = 2;\n"
         lines << "#{indent}  }\n"
         lines << "#{indent}}\n"
-        positions.each { |_, node, _, _| @position_temporaries.delete(node) }
+        positions.each { |_, node, _| @position_temporaries.delete(node) }
+        @guarded_axes.delete(write)
         lines
       end
 
+      # Which axis the nth guarded position belongs to.
+      def positions_axis (write, nth)
+        guarded_axes_of(write).fetch(nth)
+      end
+
+      def guarded_axes_of (write)
+        write_subscripts(write).each_with_index.filter_map { |(index, offset), axis|
+          axis if guarded_axis?(write, index, offset, axis)
+        }
+      end
+
+      # [temporary, position, extent text] for each axis of a write that is
+      # addressed at a position only the running kernel knows.
       def scattered_positions (write)
         subscripts = write_subscripts(write)
         subscripts.each_with_index.filter_map { |(index, offset), axis|
-          next unless index.nil? && offset.is_a?(Node) &&
-                      !Analyzer.fixed_subscript?(offset)
-          [next_temporary("position"), offset, write.array, axis]
+          # An axis whose length the kernel works out has nothing that was
+          # settled as the block was read -- a number, an index, an index
+          # with an offset, all of them -- so every write onto one works its
+          # position out here and is tested before it lands.
+          next unless guarded_axis?(write, index, offset, axis)
+          position = if index.nil? && offset.is_a?(Node) then offset
+                     elsif index.nil? then IntegerLiteral.new(offset)
+                     elsif offset.is_a?(Node) then offset
+                     else index_position_node(index, offset)
+                     end
+          [next_temporary("position"), position, write_extent(write, axis)]
         }
       end
 
+      # `k`, `k + 1`: the position an index and its offset stand for, as a
+      # node, so that a write onto an axis the kernel sized can work it out
+      # into a temporary and test it the way a scatter does.
+      def index_position_node (index, offset)
+        read = IndexVariable.new(index, 0)
+        read.type = :int64
+        return read if offset.zero?
+        built = BinaryOperation.new(offset.negative? ? :- : :+, read,
+                                    IntegerLiteral.new(offset.abs))
+        built.type = :int64
+        built
+      end
+
+      def sized_at_runtime? (write, axis)
+        write.respond_to?(:shape) && !write.shape.fetch(axis).is_a?(Integer)
+      end
+
+      def guarded_axis? (write, index, offset, axis)
+        return true if sized_at_runtime?(write, axis)
+        index.nil? && offset.is_a?(Node) && !Analyzer.fixed_subscript?(offset)
+      end
+
+      # The extent an axis of a write is tested against.  An operand's is a
+      # variable the kernel was handed; an array the body made carries the
+      # length the block wrote, and there is no variable to name.
+      def write_extent (write, axis)
+        case write
+        when LocalArrayWrite, LocalArrayMaskWrite
+          local_array_extent_text(write, axis)
+        else extent_name(write.array, axis)
+        end
+      end
+
       # `out[i] = UNDEF` marks the cell missing.  The bytes underneath are out
       # of contract, so there is nothing to store into them.
       def emit_mask_only_write (write, indent)
@@ -1267,8 +1945,23 @@ class CArray
       end
 
       # The loop a reduction runs in, inside one cell of the outer loops.
+      #
+      # A failure that reports and carries on -- a division handed back a
+      # zero, a computed index handed back cell 0 -- is stopped here, at the
+      # head of the next pass, in a function as in a kernel.  The flag is
+      # spelled three ways: a kernel's slot, the flag standing in a function's
+      # own object, and the pointer a pasted body is handed, which a kernel
+      # passes as null under a masked cell and so is asked about first.
+      #
+      # Called once the loop's body is emitted, since the body is what says
+      # whether there is anything to report.
       def inner_loop_guard (indent)
-        @body_reports ? "#{indent}  if ( *error ) break;\n" : ""
+        return "" unless @body_reports
+        flag = if !@in_function then "*error"
+               elsif @error_parameter then "#{ERROR_FLAG} && *#{ERROR_FLAG}"
+               else ERROR_FLAG
+               end
+        "#{indent}  if ( #{flag} ) break;\n"
       end
 
       # The same shape the bounded loop gets, minus the counter -- including
@@ -1294,13 +1987,13 @@ class CArray
         @carried_masks =
           @masked ? (outer + [emit_mask(loop_node.condition)]).uniq : outer
 
-        text = "#{indent}while (#{emit(loop_node.condition, :boolean)}) {\n" +
-               inner_loop_guard(indent)
-        text += loop_node.statements.map { |statement|
+        condition = emit(loop_node.condition, :boolean)
+        body = loop_node.statements.map { |statement|
           emit_statement(statement, indent + "  ")
         }.join
         @carried_masks = outer
-        text + "#{indent}}\n"
+        "#{indent}while (#{condition}) {\n" + inner_loop_guard(indent) +
+          body + "#{indent}}\n"
       end
 
       def emit_inner_loop (loop_node, indent)
@@ -1308,13 +2001,15 @@ class CArray
         return emit_split_reduction(loop_node, accumulation, indent) if accumulation
 
         index = loop_node.index
-        text = "#{indent}for (int64_t #{index} = #{emit(loop_node.from, :int64)}; " \
-               "#{index} < #{emit(loop_node.to, :int64)}; #{index}++) {\n" +
-               inner_loop_guard(indent)
-        text += loop_node.statements.map { |statement|
-          emit_statement(statement, indent + "  ")
-        }.join
-        text + "#{indent}}\n"
+        step = loop_node.step
+        opening = "#{indent}for (int64_t #{index} = #{emit(loop_node.from, :int64)}; " \
+                  "#{index} #{step.positive? ? '<' : '>'} " \
+                  "#{emit(loop_node.to, :int64)}; #{stride_step(index, step)}) {\n"
+        body = local_declarations(loop_node, indent + "  ") +
+               loop_node.statements.map { |statement|
+                 emit_statement(statement, indent + "  ")
+               }.join
+        opening + inner_loop_guard(indent) + body + "#{indent}}\n"
       end
 
       # The loop is a reduction when its whole body is one local folding a
@@ -1329,14 +2024,20 @@ class CArray
         # A masked accumulator carries a mask beside its value, and a partial
         # sum would need one each.  Left out rather than half-done.
         return nil if @masked
+        # The split walks the index a round at a time from one end, so it is
+        # written for a loop that counts by one.  A stride or a downward
+        # sweep keeps the serial loop -- which costs nothing that was there
+        # before, those loops having been unwritable until now.
+        return nil unless loop_node.step == 1
         return nil unless loop_node.statements.size == 1
 
         assignment = loop_node.statements.first
         return nil unless assignment.is_a?(Assignment)
-        name = assignment.binding_name
-        # Declared before this loop, at this type, so the fold continues a
-        # value rather than starting one.
-        return nil unless @declared_locals[name] == assignment.type
+        local = assignment.local
+        # Live when this loop was entered, at this type and binding, so the
+        # fold continues a value rather than starting one.
+        return nil unless loop_node.entering&.[](assignment.name) ==
+                          [assignment.type, assignment.binding]
 
         expression = assignment.expression
         return nil unless expression.is_a?(BinaryOperation)
@@ -1351,9 +2052,9 @@ class CArray
         return nil if literal_extent(loop_node)&.<(PARTIAL_ACCUMULATORS)
 
         left, right = expression.left, expression.right
-        if accumulator?(left, name) && !mentions_local?(right, name)
+        if accumulator?(left, local) && !mentions_local?(right, local)
           [assignment, operator]
-        elsif accumulator?(right, name) && !mentions_local?(left, name)
+        elsif accumulator?(right, local) && !mentions_local?(left, local)
           [assignment, operator]
         end
       end
@@ -1376,13 +2077,14 @@ class CArray
         end
       end
 
-      def accumulator? (node, name)
-        node.is_a?(LocalRead) && node.binding_name == name
+      # `local` is [name, binding]: the variable, not how it is spelled.
+      def accumulator? (node, local)
+        node.is_a?(LocalRead) && node.local == local
       end
 
-      def mentions_local? (node, name)
-        return true if accumulator?(node, name)
-        node.children.any? { |child| mentions_local?(child, name) }
+      def mentions_local? (node, local)
+        return true if accumulator?(node, local)
+        node.children.any? { |child| mentions_local?(child, local) }
       end
 
       # The fold, run as PARTIAL_ACCUMULATORS chains instead of one, with the
@@ -1392,14 +2094,27 @@ class CArray
       # the operator's identity, so it is folded in exactly once.  Each chain
       # keeps the term's own operand order -- what is licensed here is the
       # order the *iterations* are grouped in, not the order within one.
+      # `k++`, `k--`, or the stride written out.
+      def stride_step (index, step)
+        case step
+        when 1  then "#{index}++"
+        when -1 then "#{index}--"
+        else step.positive? ? "#{index} += #{step}" : "#{index} -= #{-step}"
+        end
+      end
+
       def emit_split_reduction (loop_node, accumulation, indent)
         assignment, operator = accumulation
-        name  = assignment.binding_name
+        name  = local_c_name(assignment.name, assignment.binding)
         type  = assignment.type
         index = loop_node.index
         base  = "#{index}__base"
         limit = "#{index}__end"
-        lanes = (0...PARTIAL_ACCUMULATORS).map { |lane| "#{name}__p#{lane}" }
+        # A lane belongs to the accumulator, so its C name comes the way every
+        # local's does.
+        lanes = (0...PARTIAL_ACCUMULATORS).map { |lane|
+          local_c_name(assignment.name, assignment.binding, lane)
+        }
         identity = (operator == :* ? ONES : ZEROES).fetch(type)
 
         inner = indent + "  "
@@ -1408,22 +2123,22 @@ class CArray
         text += "#{inner}const int64_t #{limit} = #{emit(loop_node.to, :int64)};\n"
         text += "#{inner}#{COMPUTATION_C_TYPES.fetch(type)} #{lanes.first} = #{name}" +
                 lanes.drop(1).map { |lane| ", #{lane} = #{identity}" }.join + ";\n"
+        rounds = lanes.each_with_index.map { |lane, offset|
+          term = lane_expression(assignment.expression, assignment.local, offset)
+          "#{inner}  {\n" \
+          "#{inner}    const int64_t #{index} = #{base} + #{offset};\n" \
+          "#{inner}    #{lane} = #{emit(term, type)};\n" \
+          "#{inner}  }\n"
+        }.join
         text += "#{inner}for (; #{base} + #{PARTIAL_ACCUMULATORS} <= #{limit}; " \
                 "#{base} += #{PARTIAL_ACCUMULATORS}) {\n"
-        text += inner_loop_guard(inner)
-        lanes.each_with_index do |lane, offset|
-          term = lane_expression(assignment.expression, name, lane)
-          text += "#{inner}  {\n"
-          text += "#{inner}    const int64_t #{index} = #{base} + #{offset};\n"
-          text += "#{inner}    #{lane} = #{emit(term, type)};\n"
-          text += "#{inner}  }\n"
-        end
+        text += inner_loop_guard(inner) + rounds
         text += "#{inner}}\n"
         text += "#{inner}#{name} = #{combine_partials(lanes, operator)};\n"
+        tail = emit_statement(assignment, inner + "  ")
         text += "#{inner}for (int64_t #{index} = #{base}; #{index} < #{limit}; " \
                 "#{index}++) {\n"
-        text += inner_loop_guard(inner)
-        text += emit_statement(assignment, inner + "  ")
+        text += inner_loop_guard(inner) + tail
         text += "#{inner}}\n"
         text + "#{indent}}\n"
       end
@@ -1431,11 +2146,12 @@ class CArray
       # The same term, folding into one chain instead of into the
       # accumulator.  The accumulator appears once and at the top, which is
       # what reduction_accumulation checked, so only that operand moves.
-      def lane_expression (expression, name, lane)
+      def lane_expression (expression, local, lane)
         operands = [expression.left, expression.right].map { |operand|
-          next operand unless accumulator?(operand, name)
+          next operand unless accumulator?(operand, local)
           read = LocalRead.new(operand.name, operand.location)
-          read.binding_name = lane
+          read.binding = operand.binding
+          read.lane = lane
           read.type = operand.type
           read
         }
@@ -1481,26 +2197,28 @@ class CArray
       end
 
       def emit_assignment (assignment, indent)
-        name = assignment.binding_name
+        name = local_c_name(assignment.name, assignment.binding)
         type = assignment.type
-        declared = @declared_locals.key?(name)
-        @declared_locals[name] = type
 
         # A local carries a mask alongside its value, so that reading it later
-        # is the same as reading what it was computed from.
+        # is the same as reading what it was computed from -- and, as a cell
+        # written here would, the masks of the branches this assignment stands
+        # in.  An `if` or a `while` taken on a missing cell was decided by
+        # bytes that mean nothing, so what it writes means nothing either,
+        # wherever it writes it.  Without this the rule held for
+        # `out[i] = 1.0` and not for `v = 1.0; out[i] = v`, which is the
+        # spelling a search over a row takes.
         mask = if @masked
-                 "#{indent}#{declared ? '' : 'uint8_t '}" \
-                 "#{local_mask_name(name)} = #{emit_mask(assignment.expression)};\n"
+                 "#{indent}#{local_mask_name(name)} = " \
+                 "#{combine_masks(@carried_masks + [emit_mask(assignment.expression)])};\n"
                else
                  ""
                end
 
         if assignment.expression.is_a?(Conditional)
-          lines = declared ? "" : "#{indent}#{COMPUTATION_C_TYPES.fetch(type)} #{name};\n"
-          lines + emit_conditional_statement(assignment.expression, name, type, indent) + mask
+          emit_conditional_statement(assignment.expression, name, type, indent) + mask
         else
-          prefix = declared ? "" : "#{COMPUTATION_C_TYPES.fetch(type)} "
-          "#{indent}#{prefix}#{name} = #{emit(assignment.expression, type)};\n" + mask
+          "#{indent}#{name} = #{emit(assignment.expression, type)};\n" + mask
         end
       end
 
@@ -1519,6 +2237,213 @@ class CArray
         subscripts || @analyzer.index_names.map { |name| [name, 0] }
       end
 
+      # `sum(w)` / `min(w)` / `max(w)`: a call to a helper in the preamble.
+      #
+      # A call rather than a loop unrolled at the statement, because these
+      # stand in expressions -- `max(w) - min(w)` is one line with two of them
+      # -- and an expression has nowhere to put a loop.  What it costs is
+      # nothing: the helper is `static inline` and the length is a literal, so
+      # the compiler has the same code in front of it either way.
+      def intrinsic_reference (node)
+        need(node.intrinsic, node.storage, nil)
+        "#{intrinsic_helper_name(node.intrinsic, node.storage)}" \
+        "(#{local_c_name(node.name, node.binding)}, " \
+        "#{local_array_extent_text(node, 0)})"
+      end
+
+      # A length the kernel works out cannot choose a network -- the choice
+      # is made as the C is written and a network is a fixed run of
+      # comparators -- so it sorts by the helper that takes its length as an
+      # argument, which is the insertion sort.
+      def emit_intrinsic_statement (statement, indent)
+        cells = statement.shape.first
+        name = local_c_name(statement.name, statement.binding)
+        # One cell is in order already, and a network of no comparators is
+        # nothing to call.
+        return "" if cells == 1
+        unless cells.is_a?(Integer)
+          need(:sort, statement.storage, nil)
+          return "#{indent}#{intrinsic_helper_name(:sort, statement.storage)}" \
+                 "(#{name}, #{local_array_extent_text(statement, 0)});\n"
+        end
+        if SortingNetworks.covers?(cells)
+          need(:sort, statement.storage, cells)
+          "#{indent}#{intrinsic_helper_name(:sort, statement.storage)}" \
+          "_#{cells}(#{name});\n"
+        else
+          need(:sort, statement.storage, nil)
+          "#{indent}#{intrinsic_helper_name(:sort, statement.storage)}" \
+          "(#{name}, #{cells});\n"
+        end
+      end
+
+      def intrinsic_helper_name (intrinsic, storage)
+        "carray_jit_#{intrinsic}_#{storage}"
+      end
+
+      # What the preamble has to write out.  A list used as a set, the way
+      # `@clamp_types` and `@gamma_types` are, so that two sorts of one type
+      # and length are one helper.
+      def need (intrinsic, storage, cells)
+        entry = [intrinsic, storage, cells]
+        @intrinsic_needs << entry unless @intrinsic_needs.include?(entry)
+      end
+
+      # `w = CArray.double(9)` where the statement stands.  The declaration is
+      # at the head of the block; what is left here is what Ruby does at this
+      # line, which for the zeroed spellings is a fresh array of zeros.
+      #
+      # `sizeof` rather than a byte count written out: the length is already
+      # in the declaration, and one place for it is one place to be wrong.
+      # The shadow is cleared with the cells, and a cleared shadow byte is
+      # "not missing": a fresh array in Ruby holds zeros, which are values
+      # that are there.  The `empty` spelling says nothing about either, so
+      # its shadow is whatever the stack held -- the same bargain its cells
+      # are under, and the same promise the block made by writing `empty`.
+      def emit_local_array_clearing (declaration, indent)
+        return "" unless declaration.zeroed
+        name = local_c_name(declaration.name, declaration.binding)
+        # `sizeof` answers for an array and not for a pointer, so one the
+        # kernel allocated is cleared by the count it was allocated with.
+        if declaration.heap
+          cells = local_array_cells_text(name, declaration.shape)
+          bytes = "sizeof(#{storage_c_type(declaration.storage)}) * " \
+                  "(size_t) #{cells}"
+          return "#{indent}memset(#{name}, 0, #{bytes});\n" +
+            (@masked ?
+               "#{indent}memset(#{local_mask_name(name)}, 0, " \
+               "(size_t) #{cells});\n" : "")
+        end
+        "#{indent}memset(#{name}, 0, sizeof #{name});\n" +
+          (@masked ?
+             "#{indent}memset(#{local_mask_name(name)}, 0, " \
+             "sizeof #{local_mask_name(name)});\n" : "")
+      end
+
+      # `w[k] = UNDEF`: the shadow byte alone.  The cell's bytes are out of
+      # contract once it is marked, so there is nothing to store into them.
+      def emit_local_array_mask_write (write, indent)
+        "#{indent}#{local_array_reference(write, :mask)} = 1;\n"
+      end
+
+      def emit_local_array_write (write, indent)
+        target = local_array_reference(write)
+        expression = write.expression
+        # A cell of a local array carries a mask beside it the way a scalar
+        # local carries one beside its value, and by the same rule: what the
+        # expression written into it carried, and the branches the write
+        # stands in.  A write reached because a missing cell said so was
+        # reached for no reason, whichever of the three it writes.
+        mask = if @masked
+                 "#{indent}#{local_array_reference(write, :mask)} = " \
+                 "#{combine_masks(@carried_masks + [emit_mask(write.expression)])};\n"
+               else
+                 ""
+               end
+        if expression.is_a?(Conditional)
+          temporary = next_temporary
+          type = expression.type
+          "#{indent}#{COMPUTATION_C_TYPES.fetch(type)} #{temporary};\n" +
+            emit_conditional_statement(expression, temporary, type, indent) +
+            "#{indent}#{target} = " \
+            "#{cast_to_storage(temporary, type, write.storage)};\n" + mask
+        else
+          value = cast_to_storage(emit(expression, expression.type),
+                                  expression.type, write.storage)
+          "#{indent}#{target} = #{value};\n" + mask
+        end
+      end
+
+      # `w[(k) + 1]`, and for a rank above one the row-major offset with the
+      # strides folded in -- the shape is written in the block, so they are
+      # constants rather than numbers the kernel is handed.
+      # `:mask` asks for the shadow byte beside the cell rather than the
+      # cell: the same position, in the array beside it.  It reports no index
+      # of its own, for the reason a captured array's mask read does not --
+      # the value's read asks the same question a moment later, and one
+      # report is the one the cell deserves.
+      def local_array_reference (node, kind = :data)
+        name = local_c_name(node.name, node.binding)
+        name = local_mask_name(name) if kind == :mask
+        terms = node.subscripts.each_with_index.map { |(index, offset), axis|
+          position = local_array_position(node, index, offset, axis,
+                                          reporting: kind != :mask)
+          stride = local_array_stride_text(node, axis)
+          stride == "1" ? position : "(#{position}) * #{stride}"
+        }
+        "#{name}[#{terms.join(' + ')}]"
+      end
+
+      # How long an axis of a local array is, as the C says it: the number
+      # the block wrote, or the name the kernel worked the length out into.
+      def local_array_extent_text (node, axis)
+        extent = node.shape.fetch(axis)
+        return extent.to_s if extent.is_a?(Integer)
+        heap_extent_name(local_c_name(node.name, node.binding), axis)
+      end
+
+      # And how many cells it has altogether, for the allocation and the
+      # clearing.
+      def local_array_cells_text (variable, shape)
+        return shape.inject(1, :*).to_s if literal_shape?(shape)
+        heap_cells_name(variable)
+      end
+
+      # How far one step along an axis moves: the cells of every axis inside
+      # it, which are numbers where the block wrote numbers and the lengths
+      # the kernel worked out where it did not.
+      def local_array_stride_text (node, axis)
+        inner = node.shape[(axis + 1)..]
+        return inner.inject(1, :*).to_s if literal_shape?(inner)
+        inner.each_with_index.map { |extent, offset|
+          extent.is_a?(Integer) ? extent.to_s :
+            heap_extent_name(local_c_name(node.name, node.binding),
+                             axis + 1 + offset)
+        }.join(" * ")
+      end
+
+      # An axis the analyzer settled carries an index and a literal offset,
+      # and the C says so with nothing around it.  One it could not settle
+      # carries the expression, and the check is here.
+      def local_array_position (node, index, offset, axis, reporting: true)
+        settled = node.shape.fetch(axis).is_a?(Integer)
+        # The write around this has already worked the position out and
+        # tested it.
+        guarded = @guarded_axes[node]
+        return guarded.fetch(axis) if guarded&.key?(axis)
+        if index.nil?
+          return offset.to_s unless offset.is_a?(Node)
+          # A position that is a number is settled against a length that is
+          # one.  Against a length the kernel works out it is not: 3 is
+          # inside an array of 4 cells and outside one of 2, so it is checked
+          # where the cell is reached like any other.
+          return emit(offset, :int64) if settled &&
+                                         Analyzer.fixed_subscript?(offset)
+          # A write has already worked this position out and tested it.
+          held = @position_temporaries[offset]
+          return held if held
+          @uses_index_check = true
+          return "carray_jit_index(#{emit(offset, :int64)}, " \
+                 "#{local_array_extent_text(node, axis)}, " \
+                 "#{reporting ? error_argument : '(int32_t *) 0'})"
+        end
+        # An index on an axis whose length the kernel works out was not
+        # settled either: (a) had no number to compare the loop's reach
+        # against, so the reach is tested here.
+        unless settled
+          @uses_index_check = true
+          position = if offset.zero? then index.to_s
+                     elsif offset.negative? then "#{index} - #{-offset}"
+                     else "#{index} + #{offset}"
+                     end
+          return "carray_jit_index(#{position}, " \
+                 "#{local_array_extent_text(node, axis)}, " \
+                 "#{reporting ? error_argument : '(int32_t *) 0'})"
+        end
+        return index.to_s if offset.zero?
+        offset.negative? ? "#{index} - #{-offset}" : "#{index} + #{offset}"
+      end
+
       def emit_element_write (write, indent)
         unless @masked
           @masked_flag = nil
@@ -1547,13 +2472,15 @@ class CArray
         when ElementRead
           cell_reference(node.array, node.subscripts, :mask)
         when LocalRead
-          local_mask_name(node.binding_name)
+          local_mask_name(local_c_name(node.name, node.binding, node.lane))
+        when LocalArrayRead
+          local_array_reference(node, :mask)
         when Conditional
           branches = "#{emit(node.condition, :boolean)} ? " \
                      "#{emit_mask(node.consequent)} : #{emit_mask(node.alternative)}"
           combine_masks([emit_mask(node.condition), "(#{branches})"])
         when IntegerLiteral, FloatLiteral, IndexVariable, CaptureRead, MaskTest,
-             BoundsValue, ZeroLike
+             LocalArrayMaskTest, BoundsValue, ZeroLike
           "0"
         else
           combine_masks(node.children.map { |child| emit_mask(child) })
@@ -1578,14 +2505,24 @@ class CArray
           type = expression.type
           "#{indent}#{COMPUTATION_C_TYPES.fetch(type)} #{temporary};\n" +
             emit_conditional_statement(expression, temporary, type, indent) +
-            "#{indent}#{target} = #{cast_to_storage(temporary, type, write.array)};\n"
+            "#{indent}#{target} = " \
+            "#{cast_to_storage(temporary, type, array_storage(write.array))};\n"
         else
+          storage = array_storage(write.array)
+          real = REAL_STORAGE_TYPES[storage]
+          if real && rounded_real?(expression, real)
+            return "#{indent}#{target} = #{emit(expression, real)};\n"
+          end
           value = cast_to_storage(emit(expression, expression.type),
-                                  expression.type, write.array)
+                                  expression.type, storage)
           "#{indent}#{target} = #{value};\n"
         end
       end
 
+      # A float cell, by what it computes in: the storage a rounding written
+      # straight into keeps as a real number (see #rounded_real?).
+      REAL_STORAGE_TYPES = { "float64" => :double, "float32" => :float }.freeze
+
       # A conditional in statement position becomes if/else rather than a
       # ternary; it reads far better when the branches are long.
       def emit_conditional_statement (node, target, type, indent)
@@ -1596,31 +2533,230 @@ class CArray
         "#{indent}}\n"
       end
 
-      def storage_c_type (array)
-        STORAGE_C_TYPES.fetch(@storage_types.fetch(array))
+      # What a cell is held in, and the casts between that and the type the
+      # kernel computes in.
+      #
+      # Each is asked about a storage type rather than about an array's name.
+      # An operand's storage type is looked up under its name, but not every
+      # array a body reaches is an operand -- so a helper that did the lookup
+      # itself could only answer for the ones that are, and the answer for
+      # anything else would be a KeyError out of the middle of code
+      # generation.  The name that has one is looked up by #array_storage at
+      # the call site.
+      def storage_c_type (storage)
+        STORAGE_C_TYPES.fetch(storage)
+      end
+
+      # ---- the intrinsics, written into the preamble ----
+
+      # The accumulator `min` and `max` start from over an integer array,
+      # where starting at the limit of the type rather than at the first cell
+      # saves asking whether a cell has been seen yet.  A floating array
+      # starts at NaN instead, which any number displaces -- see
+      # #intrinsic_fold_helper.
+      INTRINSIC_LIMITS = {
+        :int64  => ["INT64_MAX", "INT64_MIN"],
+        :uint64 => ["UINT64_MAX", "0"],
+      }.freeze
+
+      def computation_c_type_of (storage)
+        COMPUTATION_C_TYPES.fetch(TypeAssignment.storage_type(storage))
+      end
+
+      def floating_storage? (storage)
+        type = TypeAssignment.storage_type(storage)
+        [:float, :double].include?(type)
+      end
+
+      def intrinsic_helpers
+        return "" if @intrinsic_needs.empty?
+        text = +""
+        %i[sum min max].each do |intrinsic|
+          @intrinsic_needs.select { |kind, _, _| kind == intrinsic }
+                          .map { |_, storage, _| storage }.uniq.each do |storage|
+            text << intrinsic_fold_helper(intrinsic, storage)
+          end
+        end
+        sorted = @intrinsic_needs.select { |kind, _, _| kind == :sort }
+        sorted.map { |_, storage, _| storage }.uniq.each do |storage|
+          text << intrinsic_order_helper(storage)
+        end
+        sorted.each do |_, storage, cells|
+          text << (cells ? intrinsic_network_helper(storage, cells)
+                         : intrinsic_insertion_helper(storage))
+        end
+        text
+      end
+
+      def intrinsic_fold_helper (intrinsic, storage)
+        cell = storage_c_type(storage)
+        result = computation_c_type_of(storage)
+        type = TypeAssignment.storage_type(storage)
+        if intrinsic == :sum
+          return <<~C
+            /* `sum(w)`, added in index order from the first cell: what the
+               same loop written out in Ruby adds, in the same order, so the
+               last bit is the same one.  No partial sums -- a local array is
+               small enough that splitting buys nothing and would change the
+               answer. */
+            static inline #{result}
+            carray_jit_sum_#{storage} (const #{cell} *w, int64_t n)
+            {
+              #{result} total = 0;
+              for ( int64_t k = 0; k < n; k++ ) total += w[k];
+              return total;
+            }
+
+          C
+        end
+        if floating_storage?(storage)
+          comparison = intrinsic == :min ? "<" : ">"
+          return <<~C
+            /* `#{intrinsic}(w)`, which is what CArray's own #{intrinsic} answers.
+               A cell takes the accumulator only by beating it outright, or by
+               being a number where the accumulator is still NaN: so a NaN is
+               skipped wherever it stands, an array of nothing but NaN comes
+               back NaN, and of two cells that compare equal the first is kept
+               -- which is how CArray keeps 0.0 or -0.0, whichever came first.
+               fmin and fmax answer the same but for that: C leaves which zero
+               they give up to the library.  Starting at NaN is what lets the
+               first cell in take the accumulator without a seen-yet test. */
+            static inline #{result}
+            carray_jit_#{intrinsic}_#{storage} (const #{cell} *w, int64_t n)
+            {
+              #{result} best = NAN;
+              for ( int64_t k = 0; k < n; k++ ) {
+                if ( w[k] #{comparison} best || best != best ) best = w[k];
+              }
+              return best;
+            }
+
+          C
+        end
+        limit = INTRINSIC_LIMITS.fetch(type)[intrinsic == :min ? 0 : 1]
+        comparison = intrinsic == :min ? "<" : ">"
+        <<~C
+          /* `#{intrinsic}(w)`, which is what CArray's own #{intrinsic} answers.
+             An integer type has no NaN, so the limit this starts from is only
+             ever an answer for an array with no cells. */
+          static inline #{result}
+          carray_jit_#{intrinsic}_#{storage} (const #{cell} *w, int64_t n)
+          {
+            #{result} best = #{limit};
+            for ( int64_t k = 0; k < n; k++ ) best = (w[k] #{comparison} best) ? w[k] : best;
+            return best;
+          }
+
+        C
+      end
+
+      # The order the two sorts put the cells in, in one place so that the
+      # network and the loop cannot disagree about it.
+      #
+      # A NaN comes after every number and ties with another NaN, which makes
+      # this a total preorder -- and a comparator network sorts under any of
+      # those, so the finite cells come out ascending with the NaNs behind
+      # them.  That is where CArray's own sort puts them.
+      #
+      # The min / max folds above skip a NaN, which drops a cell rather than
+      # moving it.  A fold wants that; a sort has to put every cell somewhere.
+      def intrinsic_order_helper (storage)
+        cell = storage_c_type(storage)
+        nan_term = floating_storage?(storage) ? " || (a != a && b == b)" : ""
+        <<~C
+          /* Whether `a` belongs after `b` once the cells are in order.#{
+            floating_storage?(storage) ?
+              "\n     A NaN belongs after every number, and two NaNs tie." : ""} */
+          static inline int
+          carray_jit_after_#{storage} (#{cell} a, #{cell} b)
+          {
+            return (b < a)#{nan_term};
+          }
+
+          /* One compare-exchange: the smaller of the two ends up in `low`.
+             Written as two selects rather than a branch, so the sequence a
+             network lays out has nothing in it to predict. */
+          static inline void
+          carray_jit_cx_#{storage} (#{cell} *low, #{cell} *high)
+          {
+            #{cell} a = *low, b = *high;
+            int swap = carray_jit_after_#{storage}(a, b);
+            *low = swap ? b : a;
+            *high = swap ? a : b;
+          }
+
+        C
+      end
+
+      def intrinsic_network_helper (storage, cells)
+        network = SortingNetworks.for(cells)
+        pairs = network.map { |low, high|
+          "  carray_jit_cx_#{storage}(&w[#{low}], &w[#{high}]);"
+        }.join("\n")
+        <<~C
+          /* `sort(w)` for #{cells} cells: a comparator network, which is the
+             same #{network.size} compare-exchanges every time in the same
+             order -- no branch, nothing to predict, and the smallest number
+             of comparisons known for this length. */
+          static inline void
+          carray_jit_sort_#{storage}_#{cells} (#{storage_c_type(storage)} *w)
+          {
+          #{pairs}
+          }
+
+        C
+      end
+
+      def intrinsic_insertion_helper (storage)
+        cell = storage_c_type(storage)
+        <<~C
+          /* `sort(w)` past the length the networks cover: an insertion sort,
+             which is a loop.  The tables would keep growing and what a
+             network buys shrinks -- long before this there are more
+             comparators than the machine has registers. */
+          static inline void
+          carray_jit_sort_#{storage} (#{cell} *w, int64_t n)
+          {
+            for ( int64_t k = 1; k < n; k++ ) {
+              #{cell} value = w[k];
+              int64_t j = k - 1;
+              while ( j >= 0 && carray_jit_after_#{storage}(w[j], value) ) {
+                w[j + 1] = w[j];
+                j--;
+              }
+              w[j + 1] = value;
+            }
+          }
+
+        C
       end
 
-      def cast_to_storage (text, type, array)
+      # The storage type of an operand, by the name the block reached it by.
+      def array_storage (array)
+        @storage_types.fetch(array)
+      end
+
+      def cast_to_storage (text, type, storage)
         # Storing into a boolean array normalises: CArray's boolean is a byte
         # holding 0 or 1, and a kernel is not the place that invariant stops
         # being true.
-        return "(uint8_t)((#{text}) ? 1 : 0)" if boolean_storage?(array)
-        target = storage_c_type(array)
+        return "(uint8_t)((#{text}) ? 1 : 0)" if boolean_storage?(storage)
+        target = storage_c_type(storage)
         return text if COMPUTATION_C_TYPES.fetch(type) == target
         "(#{target})(#{text})"
       end
 
       # The cast that takes a stored cell to the type the kernel computes in,
       # or nil where the two already agree.
-      def widening_cast (array, type)
+      def widening_cast (storage, type)
         return nil if type == :boolean
         wanted = COMPUTATION_C_TYPES.fetch(type)
-        return nil if storage_c_type(array) == wanted
+        return nil if storage_c_type(storage) == wanted
         "(#{wanted})"
       end
 
-      def boolean_storage? (array)
-        @storage_types.fetch(array, nil) == "boolean"
+      def boolean_storage? (storage)
+        storage == "boolean"
       end
 
       # A Ruby name is not always a C one: `Foo::TABLE` names one thing in
@@ -1630,6 +2766,143 @@ class CArray
         name.to_s.gsub("::", "__")
       end
 
+      # The identifier each name is written as where it is written on its own,
+      # as a variable of the kernel's body.
+      #
+      # `c_name` is not enough there.  The kernel's own parameters are `data`,
+      # `error`, `strides`, `bounds` -- ordinary words a block may perfectly
+      # well have used for a capture -- and C keeps a list of its own;
+      # `RESERVED_NAMES` is already both, and was already consulted for an
+      # index.  A capture was not consulted about, and a block closing over
+      # something called `data` compiled to C that redeclared the kernel's own
+      # parameter.  A leading underscore is C's to reserve as well, which is
+      # what the made-up names here start with.  And a C name that starts
+      # `carray_jit_` is the generator's, whoever wrote it in Ruby.
+      #
+      # An index is refused for this rather than moved, because its name is
+      # written in the block and read back in messages.  A capture's is not:
+      # what it is called in the C is nobody's business, so it is moved out of
+      # the way instead of the caller being asked to rename a variable in code
+      # that has nothing to do with the kernel.
+      #
+      # Worked out for the whole kernel at once, because where a moved name
+      # lands is not a question about that name alone: a block closing over
+      # both `data` and `carray_jit_name_data` would otherwise have the first
+      # move onto the second.  So every name that stays is claimed first, and
+      # a move that would collide keeps going.
+      #
+      # Only a name that had to move moves, so the C a reader sees is what it
+      # was.  A decorated name (`p_a`, `a_s0`, `m_a`) could not collide with a
+      # parameter or a keyword and is left alone.
+      def bare_names
+        @bare_names ||= begin
+          names = (@reals + @integers + @complexes + @unsigned_integers +
+                   @address_arrays + @address_functions.keys).uniq
+          taken = names.map { |name| c_name(name) }
+          names.each_with_object({}) do |name, found|
+            text = c_name(name)
+            if RESERVED_NAMES.include?(name.to_sym) || text.start_with?("_") ||
+               text.start_with?("carray_jit_")
+              # Not the bare `carray_jit_` prefix: `carray_jit_error` is the
+              # error flag, and a capture called `error` would land on it.
+              text = "carray_jit_name_#{text}"
+              text = "#{text}_" while taken.include?(text)
+              taken << text
+            end
+            found[name] = text
+          end
+        end
+      end
+
+      def bare_name (name)
+        bare_names.fetch(name) { c_name(name) }
+      end
+
+      # The names this generator writes of its own, beside the kernel's
+      # parameters and C's keywords: every decoration of every array the
+      # kernel reaches, at every axis it has, and a borrowed function's type.
+      # All of them, whether or not this kernel ends up writing each one, so
+      # that the set is settled before anything is emitted.
+      def decorated_names
+        @decorated_names ||=
+          @arrays.flat_map { |array|
+            [pointer_name(array), mask_pointer_name(array)] +
+              (0...array_rank(array)).flat_map { |axis|
+                [stride_name(array, axis), mask_stride_name(array, axis),
+                 extent_name(array, axis)]
+              }
+          } + @address_functions.keys.map { |name| c_function_type_name(name) }
+      end
+
+      # Whether a name is one the generator writes, or could.  Everything it
+      # makes up that is not a parameter, a keyword or a decoration has `__`
+      # in it or starts `carray_jit_` -- the suffixes a local's bindings take,
+      # an inner index's second name, the temporaries -- so those two are
+      # taken whole.
+      def generated_name? (text)
+        RESERVED_NAMES.include?(text.to_sym) || text.include?("__") ||
+          text.start_with?("carray_jit_") || decorated_names.include?(text)
+      end
+
+      # The C name of each local the block assigns, by its Ruby name.  A local
+      # is ordinary Ruby and may be called anything, including a name the
+      # generator writes -- `a_n0`, `error`, `int`, `x__2` -- and a local of
+      # that name declared in the kernel's body hid what it named.  So it is
+      # moved, to `carray_jit_name<K>_<name>`, K counting the moved locals from
+      # 1 in the order the body first assigns them, so the C -- and the cache
+      # key it is part of -- is the same every time.  One that does not
+      # collide keeps the name the block gave it, which is what a reader of
+      # the C looks for.
+      #
+      # The number is what keeps two moved locals apart once a suffix is on
+      # them.  A Ruby identifier cannot start with a digit, so the digits after
+      # `carray_jit_name` run to the next `_`, and a different K is a different
+      # C name whatever `__2`, `__p0` or `__mask` follows.  A C name starting
+      # `carray_jit_` is the generator's, so no capture is spelled like one.
+      def local_c_names
+        @local_c_names ||= begin
+          moved = 0
+          assigned_locals(@analyzer.body).uniq.to_h { |name|
+            text = name.to_s
+            text = "carray_jit_name#{moved += 1}_#{text}" if generated_name?(text)
+            [name, text]
+          }
+        end
+      end
+
+      # Every name the body binds, in the order it first binds them.  An array
+      # the body made is one of these: it is a local that happens to have
+      # cells, and it is moved off the generator's own names for the same
+      # reason and by the same numbering.
+      def assigned_locals (node)
+        own = case node
+              when Assignment, LocalArrayDeclaration then [node.name]
+              else []
+              end
+        own + node.children.compact.flat_map { |child| assigned_locals(child) }
+      end
+
+      # A local's C variable is its name, moved or not, and then what says
+      # which of its variables: `__2` for its second binding, `__p0` for a
+      # partial accumulator of a split fold -- `carray_jit_name_error__2`.
+      def local_c_name (name, binding, lane = nil)
+        local_c_names.fetch(name) + (binding == 1 ? "" : "__#{binding}") +
+          (lane ? "__p#{lane}" : "")
+      end
+
+      # An index is refused where a local is moved: its name is the block's
+      # text, and the messages speak of it.  The parameters and C's keywords
+      # were refused as the block was read; what is refused here needed the
+      # arrays first.
+      def refuse_generated_indices
+        taken = @analyzer.index_names.select { |name| generated_name?(c_name(name)) }
+        return if taken.empty?
+        raise Unsupported.new(
+          "#{taken.map { |name| "`#{name}`" }.join(', ')} " \
+          "#{taken.size == 1 ? 'is a name' : 'are names'} the kernel's own C " \
+          "uses, so #{taken.size == 1 ? 'it cannot be an index' : 'they cannot be indices'}")
+      end
+
       def pointer_name (array)
         "p_#{c_name(array)}"
       end
@@ -1652,6 +2925,29 @@ class CArray
       # what the kernel raises.  Nothing in the C reaches Ruby to do it: the
       # function still returns a number and touches no Ruby value, which is
       # what lets its address be handed to a library, or called off the GVL.
+      # The widths `clamp` is emitted for: the suffix its helper takes, the C
+      # type, and whether that type has a NaN to refuse.
+      CLAMP_C_TYPES = {
+        :double => ["double", "double", true],
+        :float  => ["float", "float", true],
+        :int64  => ["int64", "int64_t", false],
+        :uint64 => ["uint64", "uint64_t", false],
+      }.freeze
+
+      # The widths `Math.gamma` is emitted for: the helper's suffix, the C
+      # type, and the C function it falls through to.
+      GAMMA_C_TYPES = {
+        :double => ["double", "double", "tgamma", "modf"],
+        :float  => ["float", "float", "tgammaf", "modff"],
+      }.freeze
+
+      # Ruby answers `Math.gamma` for a small whole number from a table of
+      # exact values rather than from tgamma, so the table is written into
+      # the C -- filled from the Ruby that is compiling, the way `Math::PI`
+      # is emitted as the double Ruby would have used.  Its length is Ruby's
+      # own: the table covers 1 through 23, and tgamma has the rest.
+      GAMMA_TABLE_LIMIT = 23
+
       ERROR_FLAG = "carray_jit_error"
 
       # A computation type this generator has no C for.  Nothing reaches here
@@ -1666,18 +2962,19 @@ class CArray
       end
 
       def error_argument
+        # Asking for the slot is what says this body can report, and every
+        # place that can -- a checked subscript, the division helpers, a
+        # `raise`, a pasted function that takes the flag -- asks here.  So the
+        # loops around it learn it from the statements they hold, which are
+        # emitted before the loop's guard is written.  A function's loops ask
+        # too: they stop on its flag the way a kernel's stop on the slot.
+        @body_reports = true
         if @in_function
           @uses_error_flag = true
           # A pasted body reports into the kernel's slot, which reaches it as
           # a parameter; a standalone one into the flag in its own object.
           return @error_parameter ? ERROR_FLAG : "&#{ERROR_FLAG}"
         end
-        # Asking for the slot is what says this body can report, and every
-        # place that can -- a checked subscript, the division helpers, a
-        # `raise`, a pasted function that takes the flag -- asks here.  So the
-        # loops above learn it from the statements they will hold, which are
-        # emitted before the loop is opened.
-        @body_reports = true
         @masked_flag ? "(#{@masked_flag} ? (int32_t *) 0 : error)" : "error"
       end
 
@@ -1787,7 +3084,7 @@ class CArray
           return "#{mask_pointer_name(array)}[#{terms.join(' + ')}]"
         end
 
-        type = storage_c_type(array)
+        type = storage_c_type(array_storage(array))
         outer = (0...(count - 1)).map { |axis|
           index, offset = subscripts[axis]
           "(#{index_expression(array, axis, index, offset)}) * " \
@@ -1810,11 +3107,17 @@ class CArray
       # Emits `node` so that its value has type `target`, inserting a cast
       # only where the type actually changes.
       def emit (node, target)
+        return emit_rounded_real(node, target).first if rounded_real?(node, target)
         widen(*emit_raw(node), node.type, target).first
       end
 
       def emit_operand (node, target, parent_precedence, right_side = false)
-        text, precedence = widen(*emit_raw(node), node.type, target)
+        text, precedence =
+          if rounded_real?(node, target)
+            emit_rounded_real(node, target)
+          else
+            widen(*emit_raw(node), node.type, target)
+          end
         parenthesize(text, precedence, right_side ? parent_precedence + 1 : parent_precedence)
       end
 
@@ -1822,13 +3125,32 @@ class CArray
       # most of these on its own, but writing them down is what makes the
       # dumped source say where the type changed -- and `int64_t` to
       # `double _Complex` is a conversion worth seeing.
+      #
+      # `int64` and `uint64` are the exception the rank cannot express.
+      # They are the same width and differ only in sign, so the rank puts
+      # one above the other and a `uint64` reaching an `int64` context is a
+      # step *down* it -- left uncast, on the reasoning that C would do it
+      # anyway.  C does not: in `i < n` with `i` an `int64_t` and `n` a
+      # `size_t`, the conversion C performs is on the other operand, and a
+      # loop counting down from -1 then runs against a bound of 2**64 - 1.
+      # The compiler says so as `-Wsign-compare`, which is the warning every
+      # generated loop over a `size_t` extent carried.
       def widen (text, precedence, from, to)
         here, there = NUMERIC_RANK[from], NUMERIC_RANK[to]
-        return [text, precedence] if here.nil? || there.nil? || there <= here
+        return [text, precedence] if here.nil? || there.nil?
+        unless there > here || crosses_sign?(from, to)
+          return [text, precedence]
+        end
         ["(#{COMPUTATION_C_TYPES.fetch(to)})" \
          "#{parenthesize(text, precedence, LEAF_PRECEDENCE)}", UNARY_PRECEDENCE]
       end
 
+      def crosses_sign? (from, to)
+        from != to &&
+          TypeAssignment::INTEGER_TYPES.include?(from) &&
+          TypeAssignment::INTEGER_TYPES.include?(to)
+      end
+
       def parenthesize (text, precedence, needed)
         precedence < needed ? "(#{text})" : text
       end
@@ -1849,8 +3171,9 @@ class CArray
         when BoundsValue      then ["bounds[#{node.slot}]", LEAF_PRECEDENCE]
         when ZeroLike
           [ZEROES.fetch(node.type), LEAF_PRECEDENCE]
-        when LocalRead        then [node.binding_name.to_s, LEAF_PRECEDENCE]
-        when CaptureRead      then [c_name(node.name), LEAF_PRECEDENCE]
+        when LocalRead        then [local_c_name(node.name, node.binding, node.lane),
+                                    LEAF_PRECEDENCE]
+        when CaptureRead      then [bare_name(node.name), LEAF_PRECEDENCE]
         # A read is widened to the type the kernel computes in, because that
         # is the type the Ruby loop computes in: reading a float32 cell in
         # Ruby gives a Float, and reading an int32 cell gives an Integer that
@@ -1861,9 +3184,20 @@ class CArray
         # A boolean cell is a byte holding 0 or 1, so it is already the value
         # it stands for.  Keeping it that way on the way out is this kernel's
         # business, and cast_to_storage does it.
+        when IntrinsicCall
+          [intrinsic_reference(node), LEAF_PRECEDENCE]
+        when LocalArrayRead
+          cell = local_array_reference(node)
+          widening = widening_cast(node.storage, node.type)
+          if widening
+            ["#{widening}#{parenthesize(cell, LEAF_PRECEDENCE, UNARY_PRECEDENCE)}",
+             UNARY_PRECEDENCE]
+          else
+            [cell, LEAF_PRECEDENCE]
+          end
         when ElementRead
           cell = cell_reference(node.array, node.subscripts)
-          widening = widening_cast(node.array, node.type)
+          widening = widening_cast(array_storage(node.array), node.type)
           if widening
             ["#{widening}#{parenthesize(cell, LEAF_PRECEDENCE, UNARY_PRECEDENCE)}",
              UNARY_PRECEDENCE]
@@ -1871,6 +3205,9 @@ class CArray
             [cell, LEAF_PRECEDENCE]
           end
         when MaskTest         then emit_mask_test(node)
+        when LocalArrayMaskTest then emit_local_array_mask_test(node)
+        when NumericPredicate then emit_numeric_predicate(node)
+        when Clamp            then emit_clamp(node)
         when UnaryMinus
           operand, precedence = emit_raw(node.operand)
           ["-#{parenthesize(operand, precedence, UNARY_PRECEDENCE)}", UNARY_PRECEDENCE]
@@ -1891,7 +3228,11 @@ class CArray
         when Power        then emit_power(node)
         when MathCall then emit_math_call(node)
         when ArrayAddress
-          [c_name(node.array), LEAF_PRECEDENCE]
+          [bare_name(node.array), LEAF_PRECEDENCE]
+        when LocalArrayAddress
+          # The array's own name, which is the address: C decays it at the
+          # call, and there is no buffer for the caller to have packed.
+          [local_c_name(node.name, node.binding), LEAF_PRECEDENCE]
         when PointerRead
           # A pointer parameter is reached the way C reaches it: contiguous,
           # from the address it was handed.  No base, no stride, no bounds --
@@ -1905,7 +3246,8 @@ class CArray
           # scalars; this one is right here, so the call is direct.
           arguments = node.arguments.zip(node.parameters)
                           .map { |argument, parameter|
-                            if argument.is_a?(ArrayAddress)
+                            if argument.is_a?(ArrayAddress) ||
+                               argument.is_a?(LocalArrayAddress)
                               emit(argument, :address)
                             else
                               emit(argument, parameter.computation)
@@ -1913,11 +3255,21 @@ class CArray
                           }
           arguments << ERROR_FLAG if @error_parameter
           ["#{@own_symbol}(#{arguments.join(', ')})", LEAF_PRECEDENCE]
+        when RandomDraw
+          # The state is the address of the captured array, which the
+          # declarations above bound to a name; the draw advances it in
+          # place, which is what leaves the generator where the kernel left
+          # it.
+          symbol = CArray::Rng::DRAW_FUNCTIONS
+                     .fetch(@randoms.fetch(node.generator).generator)
+                     .fetch(node.kind)
+          ["#{symbol}(#{bare_name(node.state)})", LEAF_PRECEDENCE]
         when CFunctionCall
           c_function = @c_functions.fetch(node.name)
           arguments = node.arguments.zip(c_function.parameters)
                           .map { |argument, parameter|
-                            if argument.is_a?(ArrayAddress)
+                            if argument.is_a?(ArrayAddress) ||
+                               argument.is_a?(LocalArrayAddress)
                               emit(argument, :address)
                             else
                               emit(argument, parameter.computation)
@@ -1925,7 +3277,18 @@ class CArray
                           }
           # A pasted body is reached by its symbol; an address by the local
           # the declarations bound it to.
-          called = c_function.pasted? ? c_function.name : c_name(node.name)
+          called = if c_function.pasted?
+                     c_function.symbol
+                   elsif @in_function
+                     # Borrowed, inside a body that has no `functions` buffer
+                     # to take an address in -- so it is called by the name it
+                     # has, declared in the preamble, and the linker or the
+                     # loader resolves it.  `jit_extern` opened the library to
+                     # find it, so by here the symbol is in the process.
+                     c_function.symbol
+                   else
+                     bare_name(node.name)
+                   end
           # And one that can report a failure is handed the slot this kernel
           # is watching -- null under a masked cell, where the value written
           # is out of contract and a division by zero there was not asked
@@ -1941,11 +3304,56 @@ class CArray
       end
 
       # `a[i] == UNDEF` reads the mask byte, never the value.
+      # `Math.gamma` is Ruby's, which is tgamma with two things around it: a
+      # table of exact values for a small integer argument, and the domain
+      # error Ruby raises where tgamma answers with a NaN.
+      def emit_gamma (node)
+        type = node.type
+        unhandled_type(node, "Math.gamma") unless GAMMA_C_TYPES.key?(type)
+        @gamma_types |= [type]
+        suffix, = GAMMA_C_TYPES.fetch(type)
+        ["carray_jit_gamma_#{suffix}(#{emit(node.arguments.first, type)}, " \
+         "#{error_argument})", LEAF_PRECEDENCE]
+      end
+
+      # `x.clamp(low, high)` through a helper of its own width, so that each
+      # of the three is computed once and the two failures Ruby raises for
+      # are reported where they happen.
+      def emit_clamp (node)
+        type = node.type
+        unhandled_type(node, "clamp") unless CLAMP_C_TYPES.key?(type)
+        @clamp_types |= [type]
+        ["carray_jit_clamp_#{CLAMP_C_TYPES.fetch(type).first}(" \
+         "#{emit(node.value, type)}, #{emit(node.low, type)}, " \
+         "#{emit(node.high, type)}, #{error_argument})", LEAF_PRECEDENCE]
+      end
+
+      # `isnan` and `isfinite` are C's own, and take a number of any width.
+      # A Complex answers `finite?` the way Ruby answers it -- both parts
+      # finite -- through a helper, so that the number is computed once.
+      # An Integer is finite whatever it holds, and goes through `isfinite`
+      # all the same rather than being answered here: the read it stands for
+      # is a read, and dropping it would drop what the read reports.
+      def emit_numeric_predicate (node)
+        type = node.operand.type
+        function = node.name == :nan? ? "isnan" : "isfinite"
+        if complex_type?(type)
+          return emit_helper_call("complex_finite", [emit(node.operand, type)],
+                                  type == :float_complex)
+        end
+        ["#{function}(#{emit(node.operand, type)})", LEAF_PRECEDENCE]
+      end
+
       def emit_mask_test (node)
         cell = cell_reference(node.array, node.subscripts, :mask)
         node.negated ? ["! #{cell}", UNARY_PRECEDENCE] : [cell, LEAF_PRECEDENCE]
       end
 
+      def emit_local_array_mask_test (node)
+        cell = local_array_reference(node, :mask)
+        node.negated ? ["! #{cell}", UNARY_PRECEDENCE] : [cell, LEAF_PRECEDENCE]
+      end
+
       # Float#floor and friends hand back an Integer in Ruby, so the C rounds
       # and then narrows.  An Integer receiver is already there.
       def emit_conversion (node)
@@ -1957,7 +3365,28 @@ class CArray
                   UNARY_PRECEDENCE]
         end
         return [emit(node.operand, :double), LEAF_PRECEDENCE] if node.result_type == :double
-        ["(int64_t)#{node.name}(#{emit(node.operand, :double)})", UNARY_PRECEDENCE]
+        @uses_whole_number = true
+        ["carray_jit_whole_int64(#{node.name}(#{emit(node.operand, :double)}), " \
+         "#{error_argument})", LEAF_PRECEDENCE]
+      end
+
+      # The same rounding where the answer is wanted as a real number: the
+      # Integer Ruby answers is exact however large, and a float cell holds
+      # it as the rounded double already is, so it does not go through int64
+      # and is not held to its range.
+      def rounded_real? (node, target)
+        node.is_a?(Conversion) && node.result_type == :int64 &&
+          node.type == :int64 &&
+          !TypeAssignment::INTEGER_TYPES.include?(node.operand.type) &&
+          [:double, :float].include?(target)
+      end
+
+      def emit_rounded_real (node, target)
+        @uses_whole_number = true
+        text = "carray_jit_whole_real(#{node.name}(#{emit(node.operand, :double)}), " \
+               "#{error_argument})"
+        return [text, LEAF_PRECEDENCE] if target == :double
+        ["(float)#{text}", UNARY_PRECEDENCE]
       end
 
       # creal and cimag are the way out of the complex type; conj stays in it.
@@ -2018,7 +3447,7 @@ class CArray
          LEAF_PRECEDENCE]
       end
 
-      # cabs, fabs and llabs are three functions rather than one because C has
+      # cabs, fabs and an integer helper are three rather than one because C has
       # no generic for them, so this names each type instead of letting one of
       # them be what a type it has not heard of falls into.
       def emit_absolute_value (node)
@@ -2033,8 +3462,10 @@ class CArray
         case node.type
         when :double  then ["fabs(#{emit(node.operand, :double)})", LEAF_PRECEDENCE]
         when :float   then ["fabsf(#{emit(node.operand, :float)})", LEAF_PRECEDENCE]
-        when :int64   then ["llabs(#{emit(node.operand, :int64)})", LEAF_PRECEDENCE]
-        # An unsigned number is its own magnitude, and llabs would take it
+        when :int64
+          @uses_integer_abs = true
+          ["carray_jit_integer_abs(#{emit(node.operand, :int64)})", LEAF_PRECEDENCE]
+        # An unsigned number is its own magnitude, and a signed helper would take it
         # through a signed type on the way.
         when :uint64  then [emit(node.operand, :uint64), LEAF_PRECEDENCE]
         else unhandled_type(node, "abs")
@@ -2090,10 +3521,13 @@ class CArray
       # and dividing by one divides each part.
       #
       # Subtraction is the exception: there the zero really is subtracted, in
-      # Ruby as in C, so `z - x` and `x - z` are C's own operator.  So is
-      # `x * z`, which Ruby coerces and multiplies out in full -- which is
-      # why `2.0 * Complex(1.0, -0.0)` and `Complex(1.0, -0.0) * 2.0` do not
-      # agree with each other, and each is reproduced its own way.
+      # Ruby as in C, so `z - x` and `x - z` are C's own operator.  `x * z`
+      # is not the exception it looks like: Ruby coerces the x and multiplies
+      # out in full, as it does two Complex numbers, and a full product in
+      # Ruby is safe_mul's part by part -- a zero times an infinity a zero --
+      # where C's `*` recovers from infinities by Annex G's rules.  So both
+      # go through complex.c's comp_mul, and `2.0 * Complex(1.0, -0.0)` and
+      # `Complex(1.0, -0.0) * 2.0` still differ, each reproduced its own way.
       #
       # And a complex division is Smith's method as complex.c writes it,
       # which is not what the C library's __divdc3 computes.
@@ -2114,18 +3548,7 @@ class CArray
         if complex_type == :float_complex &&
            [:*, :/].include?(node.operator) &&
            complex_type?(node.left.type) && complex_type?(node.right.type)
-          helper = mixed_helper(node, :complex)
-          if helper
-            text, = emit_helper_call(*helper, false)
-          else
-            precedence = PRECEDENCE.fetch(node.operator)
-            # Parenthesised, because the cast binds tighter than the operator:
-            # without them it would narrow the left operand and leave the
-            # multiplication to be worked out around it.
-            text = "(#{emit_operand(node.left, :complex, precedence)} " \
-                   "#{node.operator} " \
-                   "#{emit_operand(node.right, :complex, precedence)})"
-          end
+          text, = emit_helper_call(*mixed_helper(node, :complex), false)
           return ["(float _Complex)#{text}", UNARY_PRECEDENCE]
         end
         narrow = complex_type == :float_complex
@@ -2138,6 +3561,9 @@ class CArray
       end
 
       def emit_helper_call (name, arguments, narrow)
+        COMPLEX_HELPER_NEEDS.fetch(name, []).each do |needed|
+          @complex_helpers |= [[needed, narrow]]
+        end
         @complex_helpers |= [[name, narrow]]
         prefix = narrow ? "carray_jit_f_" : "carray_jit_"
         ["#{prefix}#{name}(#{arguments.join(', ')})", LEAF_PRECEDENCE]
@@ -2168,10 +3594,20 @@ class CArray
         when :*
           if real_type?(right)
             ["complex_mul_real", [emit(left, ctype), emit(right, rtype)]]
+          elsif real_type?(left)
+            ["real_multiply_complex", [emit(left, rtype), emit(right, ctype)]]
+          else
+            ["complex_multiply", [emit(left, ctype), emit(right, ctype)]]
           end
         end
       end
 
+      # The helpers that call another, which has to be emitted above them.
+      COMPLEX_HELPER_NEEDS = {
+        "complex_multiply" => ["safe_multiply"],
+        "real_multiply_complex" => ["safe_multiply"],
+      }.freeze
+
       def complex_type? (type)
         TypeAssignment.complex?(type)
       end
@@ -2220,6 +3656,7 @@ class CArray
           return ["(float _Complex)#{text}", UNARY_PRECEDENCE]
         end
         function = node.name.to_s
+        return emit_gamma(node) if function == "gamma"
         function += "f" if node.type == :float
         arguments = node.arguments.map { |argument| emit(argument, node.type) }
         ["#{function}(#{arguments.join(', ')})", LEAF_PRECEDENCE]
@@ -2272,11 +3709,11 @@ class CArray
         if node.type == :float
           @uses_floor_modulo_float = true
           return ["carray_jit_floor_modulo_float(#{emit(node.left, :float)}, " \
-                  "#{emit(node.right, :float)})", LEAF_PRECEDENCE]
+                  "#{emit(node.right, :float)}, #{error_argument})", LEAF_PRECEDENCE]
         end
         unhandled_type(node, "%") unless node.type == :double
         ["carray_jit_floor_modulo_real(#{emit(node.left, :double)}, " \
-         "#{emit(node.right, :double)})", LEAF_PRECEDENCE]
+         "#{emit(node.right, :double)}, #{error_argument})", LEAF_PRECEDENCE]
       end
 
       def power_of_two_shift (node)
@@ -2306,9 +3743,11 @@ class CArray
         text
       end
 
+      # Spelled with `__`, which is what keeps it out of a block's way: a
+      # local with `__` in its name is moved (see `local_c_names`).
       def next_temporary (prefix = "result")
         @temporary_count += 1
-        "#{prefix}#{@temporary_count}"
+        "#{prefix}__#{@temporary_count}"
       end
 
     end