carray-jit 0.1.2 → 0.1.4

data/lib/carray/jit/call.rb

@@ -0,0 +1,68 @@
+# `CArray.jit_call`, and only what defining it needs.
+#
+# A file that calls `jit_call` has to have the method, and having it used to
+# mean loading the whole compiler -- the analyzer, the generator, the cache.
+# For a program that compiles, that is what it is.  For one whose call sites
+# were built ahead of it by carray-jit-aot, and where a provider answers
+# every one of them, it is a compiler loaded and never reached.
+#
+# So this is the light half: the declaration's parser, which says what a call
+# takes, and the provider, which may answer without compiling.  The compiler
+# is required at the first site no provider answers, and not before.
+#
+# `require "carray/jit"` still loads everything, and a program that does not
+# know about any of this sees no difference.
+
+require "carray"
+require "carray/jit/version"
+require "carray/jit/errors"
+require "carray/jit/c_function"
+
+class CArray
+
+  # Compiles the block as a C function and calls it, here, with the locals
+  # around it:
+  #
+  #   def moving_average (values, window)
+  #     n = values.elements
+  #     out = CArray.double(n)
+  #     CArray.jit_call("void (*)(double *out, const double *values, " \
+  #                     "size_t n, size_t window)") {
+  #       n.times { |i| ... }
+  #     }
+  #     out
+  #   end
+  #
+  # The declaration's parameter names are the join, and they do the work
+  # twice over: they are the body's parameters, so the block declares none,
+  # and they name the locals of the method around it that the call reads.
+  # In C a parameter's name in a prototype is decoration; here it is the
+  # whole binding, and a name with no local behind it is an error at the
+  # call rather than a `nil` inside it.
+  #
+  # What this saves over `jit_function` plus a `call` is the argument list --
+  # written out, it is a second statement of what the declaration already
+  # said, in an order nothing checks.  What it costs is reading those locals
+  # through the block's binding, measured at about 0.3 microseconds against a
+  # call that costs several.
+  #
+  # Compiled once per call site and kept, so the second call is a lookup.
+  # The block is read, never yielded to; a name it closes over that is not a
+  # parameter is refused, as it is for `jit_function`.
+  #
+  # @param prototype [String] the function's C declaration. `double (*)(...)`
+  #   is the spelling for one with no name, which is what a call needs: the
+  #   name would be written once to be looked at never.
+  # @return [Object, nil] what the function answered, or nil where it
+  #   returns `void`.
+  # @raise [CArray::JIT::Unsupported] when the block falls outside the
+  #   recognized subset, or names parameters the declaration already named.
+  def self.jit_call (prototype, &block)
+    unless block
+      raise JIT::Unsupported,
+            "`jit_call` compiles a block and calls it, and none was given"
+    end
+    JIT.call_here(prototype, block)
+  end
+
+end

data/lib/carray/jit/compiler.rb

@@ -3,6 +3,7 @@ require "fileutils"
 require "rbconfig"
 require "tmpdir"
 require "fiddle"
+require "monitor"
 
 class CArray
   module JIT
@@ -55,11 +56,32 @@ class CArray
       # only where it means something -- and Apple's linker rejects it
       # outright, which would trade a Linux crash for a macOS build that never
       # compiles at all.
+      #
+      # A compiled `jit_function` is the exception and exports no fixed name:
+      # its body, its helpers and the four words a window reads are static,
+      # and what leaves the object is the declared symbol, the shim beside it
+      # and the accessor -- all three carrying the body's own digest.  That is
+      # what lets carray-jit-aot link several of them into one library.
       SYMBOLIC =
         (RbConfig::CONFIG["host_os"] =~ /darwin|mswin|mingw|cygwin/ ?
            [] : ["-Wl,-Bsymbolic"]).freeze
 
-      FLAGS = ["-O3", "-fPIC", "-shared", "-ffp-contract=off", *SYMBOLIC].freeze
+      # -fwrapv makes signed overflow wrap, as CArray's own operators do and
+      # as the documentation promises.  Without it the compiler is free to
+      # assume a sum cannot overflow, so one kernel contradicts itself:
+      # `y = -a; y > 0 ? y : 0` stored a negative value for a = INT64_MIN.
+      FLAGS = ["-O3", "-fPIC", "-shared", "-ffp-contract=off", "-fwrapv", *SYMBOLIC].freeze
+
+      # A compiled body calls a function borrowed with `jit_extern` by name,
+      # and the library it came from is one the process opened -- which is
+      # where the name is resolved, when the object is loaded.  ELF leaves a
+      # shared object's undefined names to the loader as it is; Apple's
+      # linker refuses them unless told to, so a borrowed function from
+      # anywhere but libm (which `-lm` names) failed to link on macOS.
+      # `jit_extern` has opened the library before any body that calls it
+      # is compiled, so by the time the object loads the name is there.
+      LINK = (RbConfig::CONFIG["host_os"] =~ /darwin/ ?
+                ["-undefined", "dynamic_lookup"] : []).freeze
 
       # Kernels retained on disk.  Each costs about 17 KB, so the default is
       # roughly 9 MB -- far more than any real program compiles, but a bound
@@ -141,7 +163,9 @@ class CArray
             if defined?(Gem::Specification)
               begin
                 Gem::Specification.find_by_name("carray").version.to_s
-              rescue StandardError
+              rescue StandardError, LoadError
+                # A gem that is not installed raises Gem::MissingSpecError,
+                # which is a LoadError rather than a StandardError.
                 "none"
               end
             else
@@ -207,7 +231,21 @@ class CArray
         # kernel does not throw its object away.  The file then names the
         # first site that compiled it, which is a place the code was written
         # rather than the only one.
+        # One compile at a time in a process.  The staging path a build
+        # compiles to is named for the process, so two threads building at
+        # once wrote the same file and whichever renamed first left the other
+        # renaming a path that was gone -- a bare Errno::ENOENT out of three
+        # threads in four.  Serialising them also spares the second thread the
+        # build: it takes the lock after the first has renamed, and finds the
+        # object where a later run would.  Reentrant, so that a build reached
+        # from inside another -- which none is today -- would not stop here.
+        LOCK = Monitor.new
+
         def build (source, function_name, header: nil, flags: FLAGS)
+          LOCK.synchronize { build_locked(source, function_name, header: header, flags: flags) }
+        end
+
+        def build_locked (source, function_name, header: nil, flags: FLAGS)
           dump(header.to_s + source)
 
           directory = prepare(cache_directory)
@@ -232,13 +270,36 @@ class CArray
           # Build to a unique path and rename, so a concurrent process never
           # loads a half-written object.
           staging_path = "#{object_path}.#{Process.pid}"
-          compile(source_path, staging_path, flags)
+          handle =
+            begin
+              compile(source_path, staging_path, flags)
+              # Opened before it is published, under the name only this
+              # process knows.  Published first, the object stands in the
+              # cache for the moment between the rename and the open, and
+              # another process pruning in that moment can evict it -- prune
+              # spares what the process running it is building, which says
+              # nothing about what anyone else is.  The build then failed to
+              # load what it had just compiled.  Opened first, the image is
+              # mapped and stays mapped whatever the directory does
+              # afterwards, which is why evicting an object in use is safe.
+              load_new_object(staging_path)
+            rescue Exception
+              # A build that did not finish leaves nothing behind.  The
+              # source is half an entry: nothing looks it up -- a hit is an
+              # object -- and prune globs objects too, so it would sit in the
+              # cache for good, one per kernel per run against a toolchain
+              # that cannot compile.  The staging file would be swept in five
+              # minutes; it goes now, beside the source it came from.
+              remove_entry(object_path)
+              unlink(staging_path)
+              raise
+            end
           File.rename(staging_path, object_path)
           sweep_staging(directory)
           sweep_environments
           prune(directory, object_path)
 
-          [load_new_object(object_path), true]
+          [handle, true]
         end
 
         # The flags are part of the key: the same C built two ways is two
@@ -369,13 +430,28 @@ class CArray
         # this user.
         def prepare (directory)
           FileUtils.mkdir_p(directory, :mode => 0700) unless File.directory?(directory)
-          mode = File.stat(directory).mode
-          if (mode & 0022) != 0
+          stat = File.stat(directory)
+          if (stat.mode & 0022) != 0
             raise CompilationError,
                   "#{directory} is writable by other users; " \
                   "carray-jit loads shared objects from it. " \
                   "Fix its permissions or set CARRAY_JIT_CACHE elsewhere."
           end
+          # And whose it is, not only who may write to it.  A directory of
+          # someone else's is one they write to whatever its mode says, and
+          # 0700 is what an attacker's own directory would be: a cache root
+          # under a shared /tmp, where this user has yet to create the
+          # directory for their environment, is a name someone else can take
+          # first -- as a directory of their own, or as a symlink to one,
+          # which the mode is then read through.  Followed on purpose: a
+          # cache root symlinked to another volume is an ordinary
+          # arrangement, and it is the owner that says whether it is ours.
+          unless stat.uid == Process.euid
+            raise CompilationError,
+                  "#{directory} belongs to uid #{stat.uid}, not to this user " \
+                  "(uid #{Process.euid}); carray-jit loads shared objects " \
+                  "from it. Set CARRAY_JIT_CACHE to a directory of your own."
+          end
           directory
         end
 
@@ -445,13 +521,13 @@ class CArray
 
         def remove_entry (object_path)
           source_path = object_path.sub(/#{Regexp.escape(shared_object_suffix)}\z/, ".c")
-          [object_path, source_path].each do |path|
-            begin
-              File.unlink(path) if File.exist?(path)
-            rescue StandardError
-              nil
-            end
-          end
+          [object_path, source_path].each { |path| unlink(path) }
+        end
+
+        def unlink (path)
+          File.unlink(path) if File.exist?(path)
+        rescue StandardError
+          nil
         end
 
         def touch (path)
@@ -473,7 +549,19 @@ class CArray
         end
 
         def compile (source_path, object_path, flags = FLAGS)
-          command = [compiler_command, *flags, source_path, "-o", object_path, "-lm"]
+          # Asked before the spawn, because a spawn that finds nothing raises
+          # Errno::ENOENT naming the program and nothing else -- not what it
+          # was wanted for, and not where to say so.  This is the same lookup
+          # the key already makes to identify the compiler.
+          unless resolve_compiler(compiler_command)
+            raise CompilationError,
+                  "`#{compiler_command}` is not an executable this process " \
+                  "can find; carray-jit compiles its kernels with it. " \
+                  "Set CARRAY_JIT_CC to a C compiler, or leave it unset for " \
+                  "the one Ruby was built with."
+          end
+          command = [compiler_command, *flags, source_path, "-o", object_path,
+                     "-lm", *LINK]
           output = IO.popen(command, err: [:child, :out]) { |io| io.read }
           unless $?.success?
             raise CompilationError, "#{command.join(' ')}\n#{output}"

data/lib/carray/jit/expression.rb

@@ -11,8 +11,15 @@ class CArray
     # nothing here restates any of that: it substitutes operands into bodies
     # it was given and wraps the result in a loop.
     #
-    # Declining is ordinary.  An expression this cannot address -- an operand
-    # that is not laid out end to end, a data type with no C to write it in --
+    # An operand need not be laid out end to end.  A slice, a transpose or a
+    # broadcast column is read where it lies, by the byte strides CArray
+    # lends for it -- a broadcast axis is a stride of 0 -- and a view with no
+    # strides of its own is lent as a packed copy of it.  The strides are
+    # handed over at the call, so one kernel serves every view of the same
+    # kind; which operands are read that way is part of what it is cached by.
+    #
+    # Declining is ordinary.  An expression this cannot address -- a data
+    # type with no C to write it in, a destination whose cells overlap --
     # goes back to CArray, which walks it and arrives at the same answer.
     class Expression
 
@@ -54,14 +61,22 @@ class CArray
       def call (plan, out)
         return false unless C_TYPES.key?(plan.data_type)
         aliased = plan.leaves.any? { |array| array.equal?(out) }
-        kernel = kernel_for(plan, aliased) or return false
+        # A shifted read reaches cells other than the one being written, so
+        # over its own output it would read what it has just written.
+        return false if aliased && shifted_leaves(plan).any? { |i| plan.leaves[i].equal?(out) }
+        return false unless out.dim == plan.dim
         arrays = [out, *plan.leaves]
         writable = [true, *Array.new(plan.leaves.size, false)]
         Access.open(arrays, writable) do |bases|
-          return false unless bases.each_with_index.all? { |basis, i|
-            basis[:strides] == end_to_end(arrays[i])
-          }
-          kernel.call(out.elements, *pointers(plan, bases))
+          strided = strided_slots(arrays, bases) or return false
+          kernel = kernel_for(plan, aliased, strided) or return false
+          # Held here so that the buffers outlive the call.
+          dim = out.dim.pack("q*")
+          steps = strided.flat_map { |slot| slot_strides(bases, slot) }.pack("q*")
+          args = [out.elements]
+          args << Fiddle::Pointer[dim] if walks_axes?(plan, strided)
+          args << Fiddle::Pointer[steps] unless strided.empty?
+          kernel.call(*args, *pointers(plan, bases))
         end
         true
       rescue ZeroDivisionError
@@ -88,27 +103,46 @@ class CArray
       private
 
       # An expression of the same shape compiles to the same kernel whatever
-      # arrays it is over, which is what the plan's signature says.
-      def kernel_for (plan, aliased)
-        @kernels.fetch([plan.signature, aliased]) do
-          @kernels[[plan.signature, aliased]] = compile(plan, aliased)
+      # arrays it is over, which is what the plan's signature says -- given
+      # the same operands read by strides.  The strides themselves are
+      # arguments, not part of the kernel.
+      def kernel_for (plan, aliased, strided)
+        key = [plan.signature, aliased, strided]
+        @kernels.fetch(key) do
+          @kernels[key] = compile(plan, aliased, strided)
         end
       end
 
-      def compile (plan, aliased)
+      def compile (plan, aliased, strided)
+        @strided = strided
         source = source_for(plan, aliased) or return nil
         handle, = Compiler.build(source, "carray_jit_expression", flags: FLAGS)
         Fiddle::Function.new(handle["carray_jit_expression"],
                              [Fiddle::TYPE_LONG_LONG] +
-                               [Fiddle::TYPE_VOIDP] * (1 + arity(plan)),
+                               [Fiddle::TYPE_VOIDP] * (1 + arity(plan, strided)),
                              Fiddle::TYPE_VOID)
       rescue CompilationError
         nil
       end
 
-      def arity (plan)
+      def arity (plan, strided)
         plan.leaves.size + plan.leaves.count { |array| array.has_mask? } +
-          (plan.masked ? 1 : 0)
+          (plan.masked ? 1 : 0) + (walks_axes?(plan, strided) ? 1 : 0) +
+          (strided.empty? ? 0 : 1)
+      end
+
+      # A plan that reads an operand shifted, or by strides, needs to know
+      # where along each axis the cell is; any other is one flat loop.
+      def walks_axes? (plan, strided)
+        shifted?(plan) || ! strided.empty?
+      end
+
+      def shifted? (plan)
+        plan.nodes.any? { |node| node.is_a?(CArray::Fusion::Shifted) }
+      end
+
+      def shifted_leaves (plan)
+        plan.nodes.grep(CArray::Fusion::Shifted).map(&:index)
       end
 
       def pointers (plan, bases)
@@ -121,23 +155,80 @@ class CArray
         args
       end
 
-      def end_to_end (array)
-        steps = Array.new(array.ndim)
-        step = array.bytes
-        (array.ndim - 1).downto(0) do |axis|
+      def end_to_end (dim, bytes)
+        steps = Array.new(dim.size)
+        step = bytes
+        (dim.size - 1).downto(0) do |axis|
           steps[axis] = step
-          step *= array.dim[axis]
+          step *= dim[axis]
         end
         steps
       end
 
+      # The C names of the pointers that are not laid out end to end, in the
+      # order their strides are handed over: the destination, its mask, then
+      # each operand and its mask.  Nil where the destination has two cells
+      # at one address -- writing them one at a time is not what the walk
+      # does -- or where an operand is not the shape of the destination.
+      def strided_slots (arrays, bases)
+        slots = []
+        bases.each_with_index do |basis, i|
+          return nil unless basis[:dim] == arrays[0].dim
+          data, mask = i == 0 ? ["out", "out_mask"] : ["a#{i - 1}", "k#{i - 1}"]
+          if i == 0
+            return nil unless one_to_one?(basis[:strides], basis[:dim], basis[:bytes])
+            return nil if basis[:mask_strides] && ! one_to_one?(basis[:mask_strides], basis[:dim], 1)
+          end
+          slots << data unless basis[:strides] == end_to_end(basis[:dim], basis[:bytes])
+          if basis[:mask_strides] && basis[:mask_strides] != end_to_end(basis[:dim], 1)
+            slots << mask
+          end
+        end
+        slots.freeze
+      end
+
+      def slot_strides (bases, slot)
+        index = slot.start_with?("out") ? 0 : slot[1..].to_i + 1
+        slot.start_with?("k", "out_mask") ? bases[index][:mask_strides] : bases[index][:strides]
+      end
+
+      # Whether every cell has an address of its own.  Taken axis by axis
+      # from the shortest stride up, each has to step past everything the
+      # ones below it reach.  A layout that interleaves axes without sharing
+      # an address fails this too, and is declined though it need not be.
+      def one_to_one? (strides, dim, bytes)
+        reach = bytes
+        strides.zip(dim).reject { |_, d| d == 1 }.sort_by { |s, _| s.abs }.all? do |s, d|
+          fits = s.abs >= reach
+          reach += s.abs * (d - 1)
+          fits
+        end
+      end
+
+      # A cell of the array `slot` names: at the flat index `flat` where it
+      # is laid out end to end, and at the sum of `axes` times its strides
+      # where it is not.
+      def cell (slot, type, flat, axes)
+        return "#{slot}[#{flat}]" unless @strided.include?(slot)
+        offset = axes.each_with_index.map { |a, k| "(#{a}) * s_#{slot}_#{k}" }.join(" + ")
+        qualifier = slot.start_with?("out") ? "" : "const "
+        "*(#{qualifier}#{type} *) ((#{qualifier}char *) #{slot} + #{offset})"
+      end
+
+      def axes (plan)
+        (0...plan.dim.size).map { |k| "i#{k}" }
+      end
+
       # -- the C ------------------------------------------------------------
 
       def source_for (plan, aliased)
         out_type = C_TYPES.fetch(plan.data_type)
         restrict = aliased ? "" : "restrict "
-        body = plan.nodes.each_with_index.map { |node, i| line(plan, node, i) }
-        return nil if body.any?(&:nil?)
+        body = statements(plan, :edge) or return nil
+        last = plan.nodes.size - 1
+        result = ["#{cell("out", out_type, "n", axes(plan))} = v#{last};",
+                  *(plan.masked ? ["#{cell("out_mask", "uint8_t", "n", axes(plan))} = m#{last};"] : [])]
+        walking = walks_axes?(plan, @strided)
         <<~C
           #include <math.h>
           #include <stdlib.h>
@@ -153,17 +244,92 @@ class CArray
           typedef float  float32_t;
           typedef double float64_t;
 
+          #{helpers}
           void
-          carray_jit_expression (int64_t elements, #{out_type} *#{restrict}out#{mask_parameter(plan, restrict)}#{parameters(plan, restrict)})
+          carray_jit_expression (int64_t elements#{walking ? ", const int64_t *dim" : ""}#{@strided.empty? ? "" : ", const int64_t *steps"}, #{out_type} *#{restrict}out#{mask_parameter(plan, restrict)}#{parameters(plan, restrict)})
           {
-            for ( int64_t n = 0; n < elements; n++ ) {
-          #{body.flatten.map { |l| "    " + l }.join("\n")}
-              out[n] = v#{plan.nodes.size - 1};#{plan.masked ? "\n    out_mask[n] = m#{plan.nodes.size - 1};" : ""}
-            }
+          #{stride_lines(plan).join("\n")}
+          #{loop(plan, body + result, walking && (statements(plan, :inside) + result)).join("\n")}
           }
         C
       end
 
+      # The C for every node, in order.  `where` matters only to a shifted
+      # read: at the :edge it may fall outside the array, :inside it cannot.
+      def statements (plan, where)
+        @where = where
+        body = plan.nodes.each_with_index.map { |node, i| line(plan, node, i) }
+        body.any?(&:nil?) ? nil : body.flatten
+      end
+
+      # The strides handed over, one per axis for each slot, named.
+      def stride_lines (plan)
+        ndim = plan.dim.size
+        @strided.each_with_index.flat_map { |slot, j|
+          (0...ndim).map { |k| "  const int64_t s_#{slot}_#{k} = steps[#{j * ndim + k}];" }
+        }
+      end
+
+      # Cell by cell in storage order.  A plan that reads an array shifted
+      # or by strides needs to know where along each axis the cell is, so it
+      # walks the outer axes and then the last.  A shifted read splits the
+      # last one in three: the stretch in the middle, where no shifted read
+      # can leave the array, is a plain loop the compiler can vectorise, and
+      # only the ends check.
+      def loop (plan, edge, inside = nil)
+        unless inside
+          return ["  for ( int64_t n = 0; n < elements; n++ ) {",
+                  *edge.map { |l| "    " + l }, "  }"]
+        end
+        ndim = plan.dim.size
+        last = ndim - 1
+        offsets = plan.nodes.grep(CArray::Fusion::Shifted).map(&:offset)
+        lines = (0...ndim).map { |k| "  const int64_t d#{k} = dim[#{k}];" }
+        plan.nodes.each_with_index do |node, i|
+          next unless node.is_a?(CArray::Fusion::Shifted)
+          flat = (1...ndim).reduce("(int64_t)(#{node.offset[0]})") { |acc, k|
+            "(#{acc}) * d#{k} + (#{node.offset[k]})"
+          }
+          lines << "  const int64_t o#{i} = #{flat};"
+        end
+        (0...ndim).each do |k|
+          break if offsets.empty?
+          below = [0, *offsets.map { |o| -o[k] }].max
+          above = [0, *offsets.map { |o| o[k] }].max
+          lines << "  const int64_t lo#{k} = d#{k} < #{below} ? d#{k} : #{below};"
+          lines << "  const int64_t hi#{k} = d#{k} - #{above} > lo#{k} ? d#{k} - #{above} : lo#{k};"
+        end
+        indent = "  "
+        (0...last).each do |k|
+          lines << "#{indent}for ( int64_t i#{k} = 0; i#{k} < d#{k}; i#{k}++ ) {"
+          indent += "  "
+        end
+        row = (0...last).reduce("(int64_t) 0") { |acc, k| "(#{acc}) * d#{k} + i#{k}" }
+        row = "(#{row}) * d#{last}"
+        outer = (0...last).map { |k| "i#{k} >= lo#{k} && i#{k} < hi#{k}" }
+        lines << "#{indent}const int64_t row = #{row};"
+        unless offsets.empty?
+          lines << "#{indent}const int64_t a = #{outer.empty? ? "lo#{last}" : "(#{outer.join(" && ")}) ? lo#{last} : d#{last}"};"
+          lines << "#{indent}const int64_t b = a < hi#{last} ? hi#{last} : a;"
+        end
+        stretches = if offsets.empty?
+                      [["0", "d#{last}", edge]]
+                    else
+                      [["0", "a", edge], ["a", "b", inside], ["b", "d#{last}", edge]]
+                    end
+        stretches.each do |from, to, body|
+          lines << "#{indent}for ( int64_t i#{last} = #{from}; i#{last} < #{to}; i#{last}++ ) {"
+          lines << "#{indent}  const int64_t n = row + i#{last};"
+          lines.concat(body.map { |l| "#{indent}  " + l })
+          lines << "#{indent}}"
+        end
+        (last - 1).downto(0) do |k|
+          indent = indent[2..]
+          lines << "#{indent}}"
+        end
+        lines
+      end
+
       def mask_parameter (plan, restrict)
         plan.masked ? ", uint8_t *#{restrict}out_mask" : ""
       end
@@ -180,31 +346,100 @@ class CArray
         type = C_TYPES[node.data_type] or return nil
         case node
         when CArray::Fusion::Leaf
-          ["#{type} v#{i} = a#{node.index}[n];",
-           *(plan.masked ? ["uint8_t m#{i} = #{node.masked ? "k#{node.index}[n]" : "0"};"] : [])]
+          ["#{type} v#{i} = #{cell("a#{node.index}", type, "n", axes(plan))};",
+           *(plan.masked ? ["uint8_t m#{i} = #{node.masked ? cell("k#{node.index}", "uint8_t", "n", axes(plan)) : "0"};"] : [])]
+        when CArray::Fusion::Shifted
+          shifted_line(plan, node, i, type)
         when CArray::Fusion::Const
           ["#{type} v#{i} = #{literal(node)};",
            *(plan.masked ? ["uint8_t m#{i} = 0;"] : [])]
         when CArray::Fusion::Op
           statement = substitute(node, i, type) or return nil
-          [*(plan.masked ? [mask_line(node, i)] : []),
+          # A mask rule this gem does not know is a plan it declines.
+          mask = plan.masked ? (mask_line(node, i) or return nil) : nil
+          [*mask,
            "#{type} v#{i};",
            *guarded(node, i, statement, plan.masked)]
         end
       end
 
+      # The cell `offset` away, read from an index clamped into the array so
+      # that the load is always in bounds, then replaced where it fell out.
+      def shifted_line (plan, node, i, type)
+        bounds = node.bounds.uniq
+        return nil unless bounds.size == 1
+        ndim = node.offset.size
+        if @where == :inside
+          moved = (0...ndim).map { |k| "i#{k} + (#{node.offset[k]})" }
+          return ["#{type} v#{i} = #{cell("a#{node.index}", type, "n + o#{i}", moved)};",
+                  *(plan.masked ? ["uint8_t m#{i} = #{node.masked ? cell("k#{node.index}", "uint8_t", "n + o#{i}", moved) : "0"};"] : [])]
+        end
+        index = (0...ndim).map { |k| "s#{i}_#{k}" }
+        lines = node.offset.each_with_index.map { |o, k|
+          "int64_t #{index[k]} = i#{k} + (#{o});"
+        }
+        inside = (0...ndim).map { |k| "#{index[k]} >= 0 && #{index[k]} < d#{k}" }.join(" && ")
+        lines << "int s#{i}_in = #{inside};"
+        (0...ndim).each { |k|
+          lines << "int64_t c#{i}_#{k} = #{index[k]} < 0 ? 0 : (#{index[k]} >= d#{k} ? d#{k} - 1 : #{index[k]});"
+        }
+        flat = (1...ndim).reduce("c#{i}_0") { |acc, k| "(#{acc}) * d#{k} + c#{i}_#{k}" }
+        clamped = (0...ndim).map { |k| "c#{i}_#{k}" }
+        lines << "#{type} v#{i} = #{cell("a#{node.index}", type, flat, clamped)};"
+        if bounds.first == :fill
+          fill = CArray::Fusion::Const.new(node.fill, node.data_type)
+          lines << "if ( ! s#{i}_in ) { v#{i} = #{literal(fill)}; }"
+        end
+        if plan.masked
+          inner = node.masked ? cell("k#{node.index}", "uint8_t", flat, clamped) : "0"
+          outer = bounds.first == :mask ? "1" : "0"
+          lines << "uint8_t m#{i} = s#{i}_in ? #{inner} : #{outer};"
+        end
+        lines
+      end
+
       def literal (node)
         case node.data_type
-        when :float64, :float32 then "%.17g" % node.value
-        when :boolean           then node.value ? 1 : 0
+        when :float64, :float32 then float_literal(node.value.to_f)
+        when :boolean           then truth(node.value) ? 1 : 0
         else node.value.to_s
         end
       end
 
+      # `%.17g` writes -0.0 as "-0", which C reads as the integer 0 negated,
+      # and writes NaN and the infinities as words C does not know.
+      def float_literal (value)
+        return "NAN" if value.nan?
+        return (value > 0 ? "INFINITY" : "(-INFINITY)") if value.infinite?
+        text = "%.17g" % value
+        text.match?(/[.e]/) ? text : "#{text}.0"
+      end
+
+      # A boolean cell as CArray stores it: 0 is false, as is false.
+      def truth (value)
+        value.is_a?(Numeric) ? value != 0 : !!value
+      end
+
+      # `<type>` names a helper by CArray's spelling of the type -- the
+      # float64_t of op_powi_float64_t -- not by its C spelling.
       def substitute (node, i, type)
         text = node.body.dup
         node.args.each_with_index { |arg, k| text = text.gsub("##{k + 1}", "v#{arg}") }
-        text.gsub("##{node.args.size + 1}", "v#{i}").gsub("<type>", type).lines.map(&:strip)
+        text.gsub("##{node.args.size + 1}", "v#{i}")
+            .gsub("<type>", carray_type(node.data_type, type)).lines.map(&:strip)
+      end
+
+      def carray_type (data_type, c_type)
+        case data_type
+        when :float32, :float64 then "#{data_type}_t"
+        else c_type
+        end
+      end
+
+      # The C a body may call beyond the standard library, as CArray hands it
+      # over.
+      def helpers
+        CArray.__kernel_helpers__
       end
 
       # A masked cell is not computed where computing it would raise: the
@@ -220,6 +455,9 @@ class CArray
         case node.mask
         when :pass  then "uint8_t m#{i} = m#{args[0]};"
         when :union then "uint8_t m#{i} = #{args.map { |a| "m#{a}" }.join(" | ")};"
+        # The condition's mask, else the mask of the operand it chooses.
+        when :select
+          "uint8_t m#{i} = m#{args[0]} | (v#{args[0]} ? m#{args[1]} : m#{args[2]});"
         when :kleene_or, :kleene_and
           known = if node.mask == :kleene_or
                     "((!m#{args[0]} && v#{args[0]}) || (!m#{args[1]} && v#{args[1]}))"