carray-jit 0.1.3 → 0.1.4

data/lib/carray/jit/c_generator.rb

@@ -363,6 +363,10 @@ class CArray
         # order the caller packs them.
         @unsigned_integers =
           carried_names.select { |name| scalar_types[name] == :uint64 }.sort
+        # A captured Integer that rides as int64 but meets a uint64, where it
+        # takes that width.  A negative one has none to take; the caller
+        # refuses it, as CArray refuses the same operand (see Kernel).
+        @unsigned_captures = []
         # Three buffers and no fourth: a capture whose type rides in none of
         # them would be packed into nothing and read as whatever the slot
         # held, so it is caught here rather than at the cell it computes
@@ -415,11 +419,13 @@ class CArray
         @uses_clamp = false
         @uses_wrap = false
         @uses_real_arg = false
+        @uses_real_power_complex = false
         @clamp_types = []
         @gamma_types = []
         @complex_helpers = []
         @uses_floor_divide = false
         @uses_floor_modulo = false
+        @uses_shift = false
         @uses_integer_power = false
         @uses_integer_abs = false
         @uses_whole_number = false
@@ -460,6 +466,7 @@ class CArray
       end
 
       attr_reader :arrays, :reals, :integers, :complexes, :unsigned_integers,
+                  :unsigned_captures,
                   :masked, :extent_slots,
                   :c_functions, :pasted_functions, :address_functions,
                   :address_arrays, :address_parameters,
@@ -517,7 +524,13 @@ class CArray
       # library handed the address calls without knowing about any of this.
       def generate_function (name, parameter_names, parameter_c_types,
                              return_c_type, return_type, error_parameter: false)
-        @own_symbol = name
+        # Standing alone, a body that turns out to be able to fail is lent
+        # under a wrapper (see #lending_wrapper), and the body itself goes
+        # under a name of its own.  Whether it can fail is known only once it
+        # has been emitted, so a recursive call is written to that name from
+        # the start and put back to `name` below where no wrapper came.
+        body_symbol = "#{name}#{BODY_SUFFIX}"
+        @own_symbol = error_parameter ? name : body_symbol
         @in_function = true
         @error_parameter = error_parameter
         @body_reports = false
@@ -532,6 +545,7 @@ class CArray
         lines = held.map { |statement| emit_statement(statement, "  ") }.join
         lines = local_declarations(@analyzer.body, "  ") + lines
         value = @returns_nothing ? nil : emit(statements.last, return_type)
+        parameter_names = parameter_names.map { |parameter| parameter_c_name(parameter) }
         parameters = parameter_names.zip(parameter_c_types)
                        .map { |parameter, type| type.declare(parameter) }
         # Passed whether or not the body turned out to use it, so that a
@@ -539,14 +553,131 @@ class CArray
         # same argument list the definition ends up declaring.
         parameters << "int32_t *#{ERROR_FLAG}" if error_parameter
         parameters = ["void"] if parameters.empty?
-        flag = @uses_error_flag && !error_parameter ? error_flag_declaration : ""
+        wrapped = @uses_error_flag && !error_parameter
+        unless wrapped || error_parameter
+          lines = lines.gsub("#{body_symbol}(", "#{name}(")
+          value = value&.gsub("#{body_symbol}(", "#{name}(")
+        end
+        flag = if wrapped
+                 error_flag_declaration + error_hook_declaration +
+                   error_standin_declaration + error_state_accessor(name)
+               else
+                 ""
+               end
         # Kept apart from the file it is compiled in: the definition alone is
         # what a kernel pastes into its own translation unit, where the
         # includes are already written and the helpers are shared.
         opening = "#{return_c_type}\n#{name} (#{parameters.join(', ')})\n{\n"
         body = lines + (@returns_nothing ? "}\n" : "  return #{value};\n}\n")
         @function_definition = opening + body
-        preamble + flag + opening + standing_gate + body
+        return preamble + flag + opening + body unless wrapped
+        preamble + flag +
+          "static #{return_c_type}\n#{body_symbol} (#{parameters.join(', ')})\n{\n" +
+          standing_gate + body + "\n" +
+          lending_wrapper(name, body_symbol, parameter_names, parameters,
+                          return_c_type)
+      end
+
+      # What the body is defined under when a wrapper is lent in its place.
+      # Static, so the object still exports one function.
+      BODY_SUFFIX = "_body".freeze
+
+      # The hook `CFunction#on_error` sets: a C function of one `void *`,
+      # and the pointer it is handed.  Beside the flag, and only where the
+      # flag is, since a body that cannot fail has nothing to call it for.
+      ERROR_HOOK = "carray_jit_on_error".freeze
+      ERROR_HOOK_DATA = "carray_jit_on_error_data".freeze
+
+      def error_hook_declaration
+        "/* Called once, by the call whose body put the flag up -- so a\n" \
+        "   library that goes on calling can be told to stop (NLopt's\n" \
+        "   nlopt_force_stop) rather than be handed stand-ins until it is\n" \
+        "   done.  Set by CFunction#on_error; null until then.  Per thread,\n" \
+        "   for the reason the flag beside it is. */\n" \
+        "static #{THREAD_LOCAL}void (*#{ERROR_HOOK}) (void *) = 0;\n" \
+        "static #{THREAD_LOCAL}void *#{ERROR_HOOK_DATA} = 0;\n\n"
+      end
+
+      # What a failed call answers C with.  Set by the window through
+      # `stand_in:`, and 0 until then, which is what this always was.
+      ERROR_STANDIN = "carray_jit_standin".freeze
+
+      def error_standin_declaration
+        "/* What a call answers with once the flag stands -- cast to whatever\n" \
+        "   this body returns.  A caller that reads the flag never looks at\n" \
+        "   it; one that reads the return value as a status does, and only\n" \
+        "   the caller knows which it is, so the window sets it.  0 until a\n" \
+        "   window says otherwise, which is what it always answered. */\n" \
+        "static #{THREAD_LOCAL}int64_t #{ERROR_STANDIN} = 0;\n\n"
+      end
+
+      # Where a thread's copy of the four above stands.  A thread-local has
+      # no address the loader can hand out -- `dlsym` answers for the symbol,
+      # not for any one thread's copy of it -- so Ruby asks the object itself,
+      # from the thread whose window it is.
+      #
+      # It is the only one of the five that is exported, and it wears the
+      # body's own name, so that this file can be linked beside another one
+      # generated the same way.  One object per body is what the cache makes
+      # and what these names once assumed; carray-jit-aot takes the C and
+      # links several into one library, where a fixed name is a duplicate
+      # symbol and the link fails.  The four it answers for are `static` for
+      # the same reason -- nothing outside this file reaches them by name,
+      # Ruby included, which is what this function is for.
+      ERROR_STATE_SUFFIX = "_error_state".freeze
+
+      # What Ruby looks the accessor up under, for a body compiled as
+      # `symbol`.
+      def self.error_state_name (symbol)
+        "#{symbol}#{ERROR_STATE_SUFFIX}"
+      end
+
+      def error_state_accessor (name)
+        "/* The four above, for the calling thread, in that order.  Called\n" \
+        "   from Ruby once per thread that opens a window on this body. */\n" \
+        "void\n#{CGenerator.error_state_name(name)} (void **out)\n{\n" \
+        "  out[0] = (void *) &#{ERROR_FLAG};\n" \
+        "  out[1] = (void *) &#{ERROR_HOOK};\n" \
+        "  out[2] = (void *) &#{ERROR_HOOK_DATA};\n" \
+        "  out[3] = (void *) &#{ERROR_STANDIN};\n}\n\n"
+      end
+
+      # The entry point the address is, for a body that can fail.  It turns
+      # a call away while the flag stands, as the body's own gate does, and
+      # calls the hook when this call is the one that raised it -- which is
+      # once, since every call after it is turned away at the top.  The hook
+      # is outside the body so that each way the flag can go up, a `raise`
+      # that leaves early or a division that runs on to the end, is the same
+      # one test afterwards.
+      #
+      # It is also the one place that says what a failed call answers C with.
+      # The body has several ways of ending -- a `raise` returns early, a
+      # division that had no divisor runs on to the end and returns whatever
+      # the zero the helper handed back added up to -- and a caller reading
+      # the return value as a status cannot be given a different answer
+      # depending on which.  So the gates inside the body go on returning
+      # zero to their own caller, which is this function or the body itself
+      # recursing, and what C is told is decided here, once, from the flag.
+      def lending_wrapper (name, body_symbol, parameter_names, parameters,
+                           return_c_type)
+        arguments = parameter_names.join(", ")
+        if @returns_nothing
+          "#{return_c_type}\n#{name} (#{parameters.join(', ')})\n{\n" \
+          "  if ( #{ERROR_FLAG} ) return;\n" \
+          "  #{body_symbol}(#{arguments});\n" \
+          "  if ( #{ERROR_FLAG} && #{ERROR_HOOK} ) " \
+          "#{ERROR_HOOK}(#{ERROR_HOOK_DATA});\n}\n"
+        else
+          standin = "(#{return_c_type}) #{ERROR_STANDIN}"
+          "#{return_c_type}\n#{name} (#{parameters.join(', ')})\n{\n" \
+          "  if ( #{ERROR_FLAG} ) return #{standin};\n" \
+          "  #{return_c_type} value = #{body_symbol}(#{arguments});\n" \
+          "  if ( #{ERROR_FLAG} ) {\n" \
+          "    if ( #{ERROR_HOOK} ) #{ERROR_HOOK}(#{ERROR_HOOK_DATA});\n" \
+          "    return #{standin};\n" \
+          "  }\n" \
+          "  return value;\n}\n"
+        end
       end
 
       attr_reader :function_definition
@@ -563,10 +694,11 @@ class CArray
       # looks like a right one.
       #
       # What the gate says is that the body does no more work.  It does not
-      # say what the call is worth: the flag is the contract, and the zero is
-      # a courtesy to a caller who never looks at it.  A `void` body has no
-      # courtesy to offer and leaves its out-parameters alone, which is the
-      # same refusal to work.
+      # say what the call is worth: the flag is the contract, and the zero
+      # goes to this body's own caller -- the lending wrapper, or the body
+      # itself recursing -- rather than to C, which is told whatever the
+      # window chose.  A `void` body has no zero to offer and leaves its
+      # out-parameters alone, which is the same refusal to work.
       #
       # It goes in the file and not in the definition, so that the form a
       # kernel pastes cannot pick it up whichever way the two forms are
@@ -593,26 +725,33 @@ class CArray
           :unsigned_power => @uses_unsigned_power,
           :floor_modulo_float => @uses_floor_modulo_float,
           :real_arg => @uses_real_arg,
+          :real_power_complex => @uses_real_power_complex,
           :clamp => @clamp_types.dup,
           :gamma => @gamma_types.dup,
           :complex => @complex_helpers.dup,
           :index_check => @uses_index_check,
           :floor_divide => @uses_floor_divide,
-          :floor_modulo => @uses_floor_modulo }
+          :floor_modulo => @uses_floor_modulo,
+          :shift => @uses_shift }
       end
 
-      # One per compiled object, so two functions never share it, and zeroed
-      # by whoever is about to look -- the same discipline the kernel's own
-      # error slot keeps.  It is written only where the value returned is
-      # already a stand-in, so a caller that never looks is no worse off than
-      # C leaves it.
+      # Per compiled object and per thread, and zeroed by whoever is about
+      # to look -- the same discipline the kernel's own error slot keeps.  It
+      # is written only where the value returned is already a stand-in, so a
+      # caller that never looks is no worse off than C leaves it.
+      #
+      # `_Thread_local` because a window is one thread's: the object is shared
+      # whether or not anybody meant to share it -- two blocks with the same
+      # text compile to one body -- and a single flag between two threads is a
+      # window neither of them can read.  It costs about half a nanosecond a
+      # call, measured, and only a body that can fail has one at all.
       def error_flag_declaration
         "/* Standing at 1 when a division had no divisor, and at the code of\n" \
         "   a `raise` in the body where one was reached -- the numbers a\n" \
         "   kernel reports through its own slot, with the same meanings.  A\n" \
         "   subscript on a pointer parameter is not checked here and does not\n" \
         "   appear: it is the caller's business, as it is in C. */\n" \
-        "int32_t #{ERROR_FLAG} = 0;\n\n"
+        "static #{THREAD_LOCAL}int32_t #{ERROR_FLAG} = 0;\n\n"
       end
 
       # The same kernel, wrapped so carray can drive it.
@@ -698,12 +837,14 @@ class CArray
           @uses_unsigned_power ||= needs[:unsigned_power]
           @uses_floor_modulo_float ||= needs[:floor_modulo_float]
           @uses_real_arg ||= needs[:real_arg]
+          @uses_real_power_complex ||= needs[:real_power_complex]
           @clamp_types |= needs[:clamp] || []
           @gamma_types |= needs[:gamma] || []
           @complex_helpers |= needs[:complex] || []
           @uses_index_check ||= needs[:index_check]
           @uses_floor_divide ||= needs[:floor_divide]
           @uses_floor_modulo ||= needs[:floor_modulo]
+          @uses_shift ||= needs[:shift]
           # A union rather than an or: two bodies may want a `sort` of one
           # type and length and a `sum` of another, and the preamble owes one
           # helper for each.  The same shape `:clamp` and `:gamma` take.
@@ -774,11 +915,8 @@ class CArray
                 return 0;
               }
               float remainder = fmodf(numerator, denominator);
-              if ( remainder != 0 ) {
-                if ( (remainder < 0) != (denominator < 0) ) remainder += denominator;
-              }
-              else {
-                remainder = copysignf(0.0f, denominator);
+              if ( remainder != 0 && (remainder < 0) != (denominator < 0) ) {
+                remainder += denominator;
               }
               return remainder;
             }
@@ -867,7 +1005,9 @@ class CArray
                not in an int64_t, where the cast is undefined -- CArray raises
                RangeError storing such an Integer, and so does this.  The real
                one is for a value that stays a number: 1e20.floor into a
-               float64 cell is 1e20 in Ruby, and does not pass through int64. */
+               float64 cell is 1e20 in Ruby, and does not pass through int64.
+               Adding zero clears the sign of a zero: (-0.3).ceil is the
+               Integer 0, which has none, where ceil(-0.3) is -0.0. */
             static inline double
             carray_jit_whole_real (double value, int32_t *error)
             {
@@ -877,7 +1017,7 @@ class CArray
               else if ( isinf(value) ) {
                 if ( error ) *error = value > 0 ? #{FLOAT_INFINITY_CODE} : #{FLOAT_NEGATIVE_INFINITY_CODE};
               }
-              return value;
+              return value + 0.0;
             }
 
             static inline int64_t
@@ -998,6 +1138,33 @@ class CArray
 
           C
         end
+        if @uses_real_power_complex
+          text << <<~C
+            /* A real power read as complex.  Ruby answers a negative base
+               raised to a power that is not an integer with a Complex, in
+               polar form: (-x)**y at the angle y*pi, exact on the axes.
+               Anything else is the real power. */
+            static double _Complex
+            carray_jit_real_power_complex (double x, double y)
+            {
+              double fi, fr, a;
+              if ( ! ( x < 0.0 ) || y == round(y) ) return CMPLX(pow(x, y), 0.0);
+              a  = pow(-x, y);
+              fr = modf(y, &fi);
+              if ( fr == 0.5 || fr == -0.5 ) {
+                int up = ( fr == 0.5 );
+                if ( ( modf(fi / 2.0, &fi) != fr ) ^ up ) a = -a;
+                return CMPLX(0.0, a);
+              }
+            #if defined(__APPLE__)
+              return CMPLX(a * __cospi(y), a * __sinpi(y));
+            #else
+              return CMPLX(a * cos(y * M_PI), a * sin(y * M_PI));
+            #endif
+            }
+
+          C
+        end
         if @uses_clamp
           text << <<~C
             /* A window that falls off the array reads the nearest cell it
@@ -1054,10 +1221,10 @@ class CArray
             /* Ruby's `%` floors with its division: the remainder carries the
                sign of the divisor, where C's `%` and fmod carry the sign of
                the dividend.  So add the divisor back when the remainder is
-               non-zero and disagrees with it in sign -- and, for floats,
-               give a zero remainder the divisor's sign, so the rule holds
-               without exception.  This mirrors CArray's own `:mod` kernel in
-               `ext/mkkernel.rb`.
+               non-zero and disagrees with it in sign.  A zero remainder keeps
+               fmod's sign, the dividend's, as Ruby's Float#% does: -4.0 % 2.0
+               is -0.0.  (CArray's own `%` gives it the divisor's sign instead;
+               a block means what it means in Ruby.)
 
                A zero divisor is Ruby's rather than CArray's: `3.0 % 0.0` raises
                ZeroDivisionError in Ruby, a float divisor as much as an integer
@@ -1070,6 +1237,7 @@ class CArray
                 if ( error ) *error = 1;
                 return 0;
               }
+              if ( denominator == -1 ) return 0;
               int64_t remainder = numerator % denominator;
               if ( remainder != 0 && ((remainder < 0) != (denominator < 0)) ) {
                 remainder += denominator;
@@ -1085,17 +1253,62 @@ class CArray
                 return 0;
               }
               double remainder = fmod(numerator, denominator);
-              if ( remainder != 0 ) {
-                if ( (remainder < 0) != (denominator < 0) ) remainder += denominator;
-              }
-              else {
-                remainder = copysign(0.0, denominator);
+              if ( remainder != 0 && (remainder < 0) != (denominator < 0) ) {
+                remainder += denominator;
               }
               return remainder;
             }
 
           C
         end
+        if @uses_shift
+          text << <<~C
+            /* Integer#<< and #>>: a negative count shifts the other way, and a
+               count of 64 or more shifts every bit out -- to 0, or to -1 for a
+               negative value shifted right.  C leaves both undefined, and a
+               left shift of a negative value too, so the left shift is made in
+               uint64_t.  A count held unsigned is brought into int64_t range
+               first; past 64 they all mean the same. */
+            static inline int64_t
+            carray_jit_shift_count (uint64_t count)
+            {
+              return count > 64 ? 64 : (int64_t) count;
+            }
+
+            static inline int64_t
+            carray_jit_shift_left (int64_t value, int64_t count)
+            {
+              if ( count >= 64 )  return 0;
+              if ( count <= -64 ) return value < 0 ? -1 : 0;
+              if ( count < 0 )    return value >> -count;
+              return (int64_t) ((uint64_t) value << count);
+            }
+
+            static inline int64_t
+            carray_jit_shift_right (int64_t value, int64_t count)
+            {
+              if ( count >= 64 )  return value < 0 ? -1 : 0;
+              if ( count <= -64 ) return 0;
+              if ( count < 0 )    return (int64_t) ((uint64_t) value << -count);
+              return value >> count;
+            }
+
+            static inline uint64_t
+            carray_jit_unsigned_shift_left (uint64_t value, int64_t count)
+            {
+              if ( count >= 64 || count <= -64 ) return 0;
+              return count < 0 ? value >> -count : value << count;
+            }
+
+            static inline uint64_t
+            carray_jit_unsigned_shift_right (uint64_t value, int64_t count)
+            {
+              if ( count >= 64 || count <= -64 ) return 0;
+              return count < 0 ? value << -count : value >> count;
+            }
+
+          C
+        end
         if @uses_floor_divide
           text << <<~C
             /* Ruby and CArray floor integer division and give the remainder the
@@ -1114,6 +1327,9 @@ class CArray
                 if ( error ) *error = 1;
                 return 0;
               }
+              /* INT64_MIN / -1 does not fit, and x86 traps on it: the
+                 quotient is the negation wrapped, as CArray answers it. */
+              if ( denominator == -1 ) return (int64_t) (0 - (uint64_t) numerator);
               int64_t quotient = numerator / denominator;
               if ( numerator % denominator != 0 && ((numerator < 0) != (denominator < 0)) ) {
                 quotient -= 1;
@@ -1528,10 +1744,8 @@ class CArray
         lines.empty? ? "" : lines.join + "\n"
       end
 
-      # Which way each axis runs is derived, not chosen: reading a cell the
-      # kernel will later write means that cell has to be reached in one
-      # particular order, and any other order would read what was never
-      # written.
+      # Which way each axis runs is the extent's step, and the loop runs that
+      # order; nothing here checks it against the cells the body reads.
       # Each axis carries a start, a limit and a step.  The step is a compiled
       # constant when it is one, because `i++` is what lets the loop vectorise
       # and that is the case nearly every kernel is in.
@@ -1678,7 +1892,9 @@ class CArray
         # A function returns whatever it returns, whatever happened -- the
         # arrangement its C caller is left with, and the one the division
         # helper already keeps by returning zero from a divide it refused.
-        # The value is not the answer; the flag says so.
+        # The value is not the answer; the flag says so.  For a body lent
+        # out, this zero reaches the wrapper rather than C, and the wrapper
+        # answers C with what the window chose.
         leaving = @in_function && !@returns_nothing ? "return 0;" : "return;"
         "#{indent}if ( #{tests.join(' && ')} ) {\n" \
         "#{indent}  #{report}\n" +
@@ -2002,14 +2218,53 @@ class CArray
 
         index = loop_node.index
         step = loop_node.step
-        opening = "#{indent}for (int64_t #{index} = #{emit(loop_node.from, :int64)}; " \
-                  "#{index} #{step.positive? ? '<' : '>'} " \
-                  "#{emit(loop_node.to, :int64)}; #{stride_step(index, step)}) {\n"
-        body = local_declarations(loop_node, indent + "  ") +
+        inner = indent + "  "
+        prelude, from, trips = loop_trips(loop_node, inner)
+        count = "#{index}__t"
+        body = local_declarations(loop_node, inner + "  ") +
                loop_node.statements.map { |statement|
-                 emit_statement(statement, indent + "  ")
+                 emit_statement(statement, inner + "  ")
                }.join
-        opening + inner_loop_guard(indent) + body + "#{indent}}\n"
+        "#{indent}{\n" + prelude +
+          "#{inner}for (uint64_t #{count} = 0; #{count} < #{trips}; #{count}++) {\n" \
+          "#{inner}  int64_t #{index} = (int64_t) ((uint64_t) #{from} + " \
+          "#{count} * (uint64_t) INT64_C(#{step}));\n" +
+          inner_loop_guard(inner) + body + "#{inner}}\n" + "#{indent}}\n"
+      end
+
+      # How many passes a loop makes, counted before it starts and in uint64.
+      # Counting by comparing the index with the end overflows at the ends of
+      # int64: an inclusive range ends at its last index plus one, and
+      # `(MAX-2..MAX).each` ended at MAX + 1, which is no int64 at all.  The
+      # analyzer writes an inclusive end, and a `step` loop's end, as that
+      # plus or minus one; read back here, the last index is used as it is.
+      #
+      # Returns [the C declaring the first index, the end and the count,
+      # the first index's C name, the count's C name].
+      def loop_trips (loop_node, inner)
+        index = loop_node.index
+        step = loop_node.step
+        stride = step.abs
+        to = loop_node.to
+        last = to.is_a?(BinaryOperation) &&
+               to.operator == (step.positive? ? :+ : :-) &&
+               to.right.is_a?(IntegerLiteral) && to.right.value == 1 ? to.left : nil
+        from_name, end_name, trips_name = "#{index}__from", "#{index}__end", "#{index}__trips"
+        ahead = step.positive? ? "(uint64_t) #{end_name} - (uint64_t) #{from_name}" :
+                                 "(uint64_t) #{from_name} - (uint64_t) #{end_name}"
+        trips =
+          if last
+            reached = step.positive? ? "#{from_name} <= #{end_name}" : "#{from_name} >= #{end_name}"
+            "(#{reached}) ? (#{ahead}) / #{stride}u + 1 : 0"
+          else
+            reached = step.positive? ? "#{from_name} < #{end_name}" : "#{from_name} > #{end_name}"
+            "(#{reached}) ? (#{ahead} - 1) / #{stride}u + 1 : 0"
+          end
+        prelude =
+          "#{inner}const int64_t #{from_name} = #{emit(loop_node.from, :int64)};\n" \
+          "#{inner}const int64_t #{end_name} = #{emit(last || to, :int64)};\n" \
+          "#{inner}const uint64_t #{trips_name} = #{trips};\n"
+        [prelude, from_name, trips_name]
       end
 
       # The loop is a reduction when its whole body is one local folding a
@@ -2109,7 +2364,6 @@ class CArray
         type  = assignment.type
         index = loop_node.index
         base  = "#{index}__base"
-        limit = "#{index}__end"
         # A lane belongs to the accumulator, so its C name comes the way every
         # local's does.
         lanes = (0...PARTIAL_ACCUMULATORS).map { |lane|
@@ -2118,26 +2372,26 @@ class CArray
         identity = (operator == :* ? ONES : ZEROES).fetch(type)
 
         inner = indent + "  "
-        text  = "#{indent}{\n"
-        text += "#{inner}int64_t #{base} = #{emit(loop_node.from, :int64)};\n"
-        text += "#{inner}const int64_t #{limit} = #{emit(loop_node.to, :int64)};\n"
+        prelude, from, trips = loop_trips(loop_node, inner)
+        text  = "#{indent}{\n" + prelude
+        text += "#{inner}uint64_t #{base} = 0;\n"
         text += "#{inner}#{COMPUTATION_C_TYPES.fetch(type)} #{lanes.first} = #{name}" +
                 lanes.drop(1).map { |lane| ", #{lane} = #{identity}" }.join + ";\n"
         rounds = lanes.each_with_index.map { |lane, offset|
           term = lane_expression(assignment.expression, assignment.local, offset)
           "#{inner}  {\n" \
-          "#{inner}    const int64_t #{index} = #{base} + #{offset};\n" \
+          "#{inner}    const int64_t #{index} = (int64_t) ((uint64_t) #{from} + #{base} + #{offset});\n" \
           "#{inner}    #{lane} = #{emit(term, type)};\n" \
           "#{inner}  }\n"
         }.join
-        text += "#{inner}for (; #{base} + #{PARTIAL_ACCUMULATORS} <= #{limit}; " \
+        text += "#{inner}for (; #{trips} - #{base} >= #{PARTIAL_ACCUMULATORS}; " \
                 "#{base} += #{PARTIAL_ACCUMULATORS}) {\n"
         text += inner_loop_guard(inner) + rounds
         text += "#{inner}}\n"
         text += "#{inner}#{name} = #{combine_partials(lanes, operator)};\n"
-        tail = emit_statement(assignment, inner + "  ")
-        text += "#{inner}for (int64_t #{index} = #{base}; #{index} < #{limit}; " \
-                "#{index}++) {\n"
+        tail = emit_statement(assignment, inner + "    ")
+        text += "#{inner}for (; #{base} < #{trips}; #{base}++) {\n"
+        text += "#{inner}  const int64_t #{index} = (int64_t) ((uint64_t) #{from} + #{base});\n"
         text += inner_loop_guard(inner) + tail
         text += "#{inner}}\n"
         text + "#{indent}}\n"
@@ -2348,8 +2602,8 @@ class CArray
             "#{indent}#{target} = " \
             "#{cast_to_storage(temporary, type, write.storage)};\n" + mask
         else
-          value = cast_to_storage(emit(expression, expression.type),
-                                  expression.type, write.storage)
+          type = complex_storage_of_power(expression, write.storage) || expression.type
+          value = cast_to_storage(emit(expression, type), type, write.storage)
           "#{indent}#{target} = #{value};\n" + mask
         end
       end
@@ -2480,7 +2734,7 @@ class CArray
                      "#{emit_mask(node.consequent)} : #{emit_mask(node.alternative)}"
           combine_masks([emit_mask(node.condition), "(#{branches})"])
         when IntegerLiteral, FloatLiteral, IndexVariable, CaptureRead, MaskTest,
-             LocalArrayMaskTest, BoundsValue, ZeroLike
+             LocalArrayMaskTest, PointerTest, BoundsValue, ZeroLike
           "0"
         else
           combine_masks(node.children.map { |child| emit_mask(child) })
@@ -2513,12 +2767,21 @@ class CArray
           if real && rounded_real?(expression, real)
             return "#{indent}#{target} = #{emit(expression, real)};\n"
           end
-          value = cast_to_storage(emit(expression, expression.type),
-                                  expression.type, storage)
+          type = complex_storage_of_power(expression, storage) || expression.type
+          value = cast_to_storage(emit(expression, type), type, storage)
           "#{indent}#{target} = #{value};\n"
         end
       end
 
+      # A real power stored in a complex cell is read as complex; see
+      # complex_read_power?.
+      COMPLEX_STORAGE_TYPES = { "cmplx128" => :complex, "cmplx64" => :float_complex }.freeze
+
+      def complex_storage_of_power (expression, storage)
+        type = COMPLEX_STORAGE_TYPES[storage]
+        type if type && complex_read_power?(expression, type)
+      end
+
       # A float cell, by what it computes in: the storage a rounding written
       # straight into keeps as a real number (see #rounded_real?).
       REAL_STORAGE_TYPES = { "float64" => :double, "float32" => :float }.freeze
@@ -2818,6 +3081,15 @@ class CArray
         bare_names.fetch(name) { c_name(name) }
       end
 
+      # A parameter the body assigns is a local of the same name from the
+      # top of the body on, so the C parameter it starts from goes under a
+      # name of its own -- one only this generator writes.  Every other
+      # name is what it always was.
+      def parameter_c_name (name)
+        return bare_name(name) unless @analyzer.assigned_parameters.include?(name.to_sym)
+        "carray_jit_parameter_#{c_name(name)}"
+      end
+
       # The names this generator writes of its own, beside the kernel's
       # parameters and C's keywords: every decoration of every array the
       # kernel reaches, at every axis it has, and a borrowed function's type.
@@ -2950,6 +3222,12 @@ class CArray
 
       ERROR_FLAG = "carray_jit_error"
 
+      # The storage class the flag and the hook beside it are declared with.
+      # C11's, which is what every compiler this gem drives is given; MSVC's
+      # `__declspec(thread)` is not written here because the build flags are
+      # already gcc's and cl has never been one of them.
+      THREAD_LOCAL = "_Thread_local ".freeze
+
       # A computation type this generator has no C for.  Nothing reaches here
       # today -- every type TypeAssignment can assign is named at each of the
       # sites that calls this -- and that is the point: a type added later
@@ -3107,13 +3385,26 @@ class CArray
       # Emits `node` so that its value has type `target`, inserting a cast
       # only where the type actually changes.
       def emit (node, target)
+        note_unsigned_capture(node, target)
+        return emit_complex_read_power(node, target).first if complex_read_power?(node, target)
         return emit_rounded_real(node, target).first if rounded_real?(node, target)
         widen(*emit_raw(node), node.type, target).first
       end
 
+      # A signed capture read where a uint64 is computed -- absorbed into one,
+      # or compared with one.  Kernel refuses a negative value for it.
+      def note_unsigned_capture (node, target)
+        return unless node.is_a?(CaptureRead) && target == :uint64 &&
+                      @integers.include?(node.name)
+        @unsigned_captures |= [node.name]
+      end
+
       def emit_operand (node, target, parent_precedence, right_side = false)
+        note_unsigned_capture(node, target)
         text, precedence =
-          if rounded_real?(node, target)
+          if complex_read_power?(node, target)
+            emit_complex_read_power(node, target)
+          elsif rounded_real?(node, target)
             emit_rounded_real(node, target)
           else
             widen(*emit_raw(node), node.type, target)
@@ -3173,7 +3464,9 @@ class CArray
           [ZEROES.fetch(node.type), LEAF_PRECEDENCE]
         when LocalRead        then [local_c_name(node.name, node.binding, node.lane),
                                     LEAF_PRECEDENCE]
-        when CaptureRead      then [bare_name(node.name), LEAF_PRECEDENCE]
+        when CaptureRead
+          note_unsigned_capture(node, node.type)
+          [parameter_c_name(node.name), LEAF_PRECEDENCE]
         # A read is widened to the type the kernel computes in, because that
         # is the type the Ruby loop computes in: reading a float32 cell in
         # Ruby gives a Float, and reading an int32 cell gives an Integer that
@@ -3207,6 +3500,8 @@ class CArray
         when MaskTest         then emit_mask_test(node)
         when LocalArrayMaskTest then emit_local_array_mask_test(node)
         when NumericPredicate then emit_numeric_predicate(node)
+        when PointerTest
+          ["#{node.name} #{node.null ? '==' : '!='} NULL", PRECEDENCE.fetch(:==)]
         when Clamp            then emit_clamp(node)
         when UnaryMinus
           operand, precedence = emit_raw(node.operand)
@@ -3411,6 +3706,24 @@ class CArray
         end
       end
 
+      # A real power read where a complex is wanted -- stored in a complex
+      # cell or joined with a complex.  pow answers a negative base raised to
+      # a power that is not an integer with NaN; Ruby answers with a Complex,
+      # and as the value is going to be complex anyway, so does the kernel.
+      # Read as a real, the power stays pow's.
+      def complex_read_power? (node, target)
+        node.is_a?(Power) && [:double, :float].include?(node.type) &&
+          complex_type?(target)
+      end
+
+      def emit_complex_read_power (node, target)
+        @uses_real_power_complex = true
+        text = "carray_jit_real_power_complex(#{emit(node.base, :double)}, " \
+               "#{emit(node.exponent, :double)})"
+        return [text, LEAF_PRECEDENCE] if target == :complex
+        ["(float _Complex)#{text}", UNARY_PRECEDENCE]
+      end
+
       # An integer power is squared out rather than sent through pow, whose
       # double result would not be the exact integer Ruby gives.
       #
@@ -3482,6 +3795,8 @@ class CArray
         return emit_integer_division(node) if node.operator == :/ &&
                                              TypeAssignment::INTEGER_TYPES.include?(node.type)
         return emit_modulo(node) if node.operator == :%
+        return emit_shift(node) if [:<<, :>>].include?(node.operator) &&
+                                   TypeAssignment::INTEGER_TYPES.include?(node.type)
         return emit_complex_binary(node) if complex_type?(node.type)
 
         operand_type =
@@ -3693,6 +4008,19 @@ class CArray
          "#{emit(node.right, :int64)}, #{error_argument})", LEAF_PRECEDENCE]
       end
 
+      # A shift is Integer's -- see carray_jit_shift_left.
+      def emit_shift (node)
+        @uses_shift = true
+        direction = node.operator == :<< ? "left" : "right"
+        prefix = node.type == :uint64 ? "carray_jit_unsigned_shift_" : "carray_jit_shift_"
+        count = if node.right.type == :uint64
+                  "carray_jit_shift_count(#{emit(node.right, :uint64)})"
+                else
+                  emit(node.right, :int64)
+                end
+        ["#{prefix}#{direction}(#{emit(node.left, node.type)}, #{count})", LEAF_PRECEDENCE]
+      end
+
       # `%` floors, so it is not C's `%` -- see the helper above.
       def emit_modulo (node)
         if node.type == :uint64
@@ -3723,11 +4051,23 @@ class CArray
         Math.log2(value).to_i
       end
 
+      # A literal past int64 is a uint64 if it fits one, and refused with the
+      # message a capture of the same value gets if it does not -- written
+      # out, it was a compiler error about a constant too large for its type.
       def format_integer (value)
-        "INT64_C(#{value})"
+        case TypeAssignment.integer_type(value)
+        when :uint64 then "UINT64_C(#{value})"
+        else "INT64_C(#{value})"
+        end
       end
 
       def format_float (value, type = :double)
+        # 1e400 is an infinity in Ruby; written out with %g it is the word
+        # `inf`, which C does not know.
+        if value.infinite?
+          text = value.positive? ? "INFINITY" : "(-INFINITY)"
+          return type == :float ? "((float)#{text})" : text
+        end
         # A float literal without the suffix is a double, and one double in an
         # expression takes the whole expression with it -- so the suffix is
         # what keeps a float32 kernel computing in float.