carray-jit 0.1.2 → 0.1.3

data/lib/carray/jit/call.rb

@@ -0,0 +1,68 @@
+# `CArray.jit_call`, and only what defining it needs.
+#
+# A file that calls `jit_call` has to have the method, and having it used to
+# mean loading the whole compiler -- the analyzer, the generator, the cache.
+# For a program that compiles, that is what it is.  For one whose call sites
+# were built ahead of it by carray-jit-aot, and where a provider answers
+# every one of them, it is a compiler loaded and never reached.
+#
+# So this is the light half: the declaration's parser, which says what a call
+# takes, and the provider, which may answer without compiling.  The compiler
+# is required at the first site no provider answers, and not before.
+#
+# `require "carray/jit"` still loads everything, and a program that does not
+# know about any of this sees no difference.
+
+require "carray"
+require "carray/jit/version"
+require "carray/jit/errors"
+require "carray/jit/c_function"
+
+class CArray
+
+  # Compiles the block as a C function and calls it, here, with the locals
+  # around it:
+  #
+  #   def moving_average (values, window)
+  #     n = values.elements
+  #     out = CArray.double(n)
+  #     CArray.jit_call("void (*)(double *out, const double *values, " \
+  #                     "size_t n, size_t window)") {
+  #       n.times { |i| ... }
+  #     }
+  #     out
+  #   end
+  #
+  # The declaration's parameter names are the join, and they do the work
+  # twice over: they are the body's parameters, so the block declares none,
+  # and they name the locals of the method around it that the call reads.
+  # In C a parameter's name in a prototype is decoration; here it is the
+  # whole binding, and a name with no local behind it is an error at the
+  # call rather than a `nil` inside it.
+  #
+  # What this saves over `jit_function` plus a `call` is the argument list --
+  # written out, it is a second statement of what the declaration already
+  # said, in an order nothing checks.  What it costs is reading those locals
+  # through the block's binding, measured at about 0.3 microseconds against a
+  # call that costs several.
+  #
+  # Compiled once per call site and kept, so the second call is a lookup.
+  # The block is read, never yielded to; a name it closes over that is not a
+  # parameter is refused, as it is for `jit_function`.
+  #
+  # @param prototype [String] the function's C declaration. `double (*)(...)`
+  #   is the spelling for one with no name, which is what a call needs: the
+  #   name would be written once to be looked at never.
+  # @return [Object, nil] what the function answered, or nil where it
+  #   returns `void`.
+  # @raise [CArray::JIT::Unsupported] when the block falls outside the
+  #   recognized subset, or names parameters the declaration already named.
+  def self.jit_call (prototype, &block)
+    unless block
+      raise JIT::Unsupported,
+            "`jit_call` compiles a block and calls it, and none was given"
+    end
+    JIT.call_here(prototype, block)
+  end
+
+end

data/lib/carray/jit/compiler.rb

@@ -3,6 +3,7 @@ require "fileutils"
 require "rbconfig"
 require "tmpdir"
 require "fiddle"
+require "monitor"
 
 class CArray
   module JIT
@@ -207,7 +208,21 @@ class CArray
         # kernel does not throw its object away.  The file then names the
         # first site that compiled it, which is a place the code was written
         # rather than the only one.
+        # One compile at a time in a process.  The staging path a build
+        # compiles to is named for the process, so two threads building at
+        # once wrote the same file and whichever renamed first left the other
+        # renaming a path that was gone -- a bare Errno::ENOENT out of three
+        # threads in four.  Serialising them also spares the second thread the
+        # build: it takes the lock after the first has renamed, and finds the
+        # object where a later run would.  Reentrant, so that a build reached
+        # from inside another -- which none is today -- would not stop here.
+        LOCK = Monitor.new
+
         def build (source, function_name, header: nil, flags: FLAGS)
+          LOCK.synchronize { build_locked(source, function_name, header: header, flags: flags) }
+        end
+
+        def build_locked (source, function_name, header: nil, flags: FLAGS)
           dump(header.to_s + source)
 
           directory = prepare(cache_directory)
@@ -232,13 +247,36 @@ class CArray
           # Build to a unique path and rename, so a concurrent process never
           # loads a half-written object.
           staging_path = "#{object_path}.#{Process.pid}"
-          compile(source_path, staging_path, flags)
+          handle =
+            begin
+              compile(source_path, staging_path, flags)
+              # Opened before it is published, under the name only this
+              # process knows.  Published first, the object stands in the
+              # cache for the moment between the rename and the open, and
+              # another process pruning in that moment can evict it -- prune
+              # spares what the process running it is building, which says
+              # nothing about what anyone else is.  The build then failed to
+              # load what it had just compiled.  Opened first, the image is
+              # mapped and stays mapped whatever the directory does
+              # afterwards, which is why evicting an object in use is safe.
+              load_new_object(staging_path)
+            rescue Exception
+              # A build that did not finish leaves nothing behind.  The
+              # source is half an entry: nothing looks it up -- a hit is an
+              # object -- and prune globs objects too, so it would sit in the
+              # cache for good, one per kernel per run against a toolchain
+              # that cannot compile.  The staging file would be swept in five
+              # minutes; it goes now, beside the source it came from.
+              remove_entry(object_path)
+              unlink(staging_path)
+              raise
+            end
           File.rename(staging_path, object_path)
           sweep_staging(directory)
           sweep_environments
           prune(directory, object_path)
 
-          [load_new_object(object_path), true]
+          [handle, true]
         end
 
         # The flags are part of the key: the same C built two ways is two
@@ -369,13 +407,28 @@ class CArray
         # this user.
         def prepare (directory)
           FileUtils.mkdir_p(directory, :mode => 0700) unless File.directory?(directory)
-          mode = File.stat(directory).mode
-          if (mode & 0022) != 0
+          stat = File.stat(directory)
+          if (stat.mode & 0022) != 0
             raise CompilationError,
                   "#{directory} is writable by other users; " \
                   "carray-jit loads shared objects from it. " \
                   "Fix its permissions or set CARRAY_JIT_CACHE elsewhere."
           end
+          # And whose it is, not only who may write to it.  A directory of
+          # someone else's is one they write to whatever its mode says, and
+          # 0700 is what an attacker's own directory would be: a cache root
+          # under a shared /tmp, where this user has yet to create the
+          # directory for their environment, is a name someone else can take
+          # first -- as a directory of their own, or as a symlink to one,
+          # which the mode is then read through.  Followed on purpose: a
+          # cache root symlinked to another volume is an ordinary
+          # arrangement, and it is the owner that says whether it is ours.
+          unless stat.uid == Process.euid
+            raise CompilationError,
+                  "#{directory} belongs to uid #{stat.uid}, not to this user " \
+                  "(uid #{Process.euid}); carray-jit loads shared objects " \
+                  "from it. Set CARRAY_JIT_CACHE to a directory of your own."
+          end
           directory
         end
 
@@ -445,13 +498,13 @@ class CArray
 
         def remove_entry (object_path)
           source_path = object_path.sub(/#{Regexp.escape(shared_object_suffix)}\z/, ".c")
-          [object_path, source_path].each do |path|
-            begin
-              File.unlink(path) if File.exist?(path)
-            rescue StandardError
-              nil
-            end
-          end
+          [object_path, source_path].each { |path| unlink(path) }
+        end
+
+        def unlink (path)
+          File.unlink(path) if File.exist?(path)
+        rescue StandardError
+          nil
         end
 
         def touch (path)
@@ -473,6 +526,17 @@ class CArray
         end
 
         def compile (source_path, object_path, flags = FLAGS)
+          # Asked before the spawn, because a spawn that finds nothing raises
+          # Errno::ENOENT naming the program and nothing else -- not what it
+          # was wanted for, and not where to say so.  This is the same lookup
+          # the key already makes to identify the compiler.
+          unless resolve_compiler(compiler_command)
+            raise CompilationError,
+                  "`#{compiler_command}` is not an executable this process " \
+                  "can find; carray-jit compiles its kernels with it. " \
+                  "Set CARRAY_JIT_CC to a C compiler, or leave it unset for " \
+                  "the one Ruby was built with."
+          end
           command = [compiler_command, *flags, source_path, "-o", object_path, "-lm"]
           output = IO.popen(command, err: [:child, :out]) { |io| io.read }
           unless $?.success?