hotcell-server 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9b2d1b6a8296d8833188c1414bfeb70d7031c8601d520361b10110e832461992
-  data.tar.gz: ec7983987f989d78eeeefee5b3fa8b5c5a860d3ae1022ce414172ea553d64938
+  metadata.gz: b0353cf7589212280f099039e6bec97f6da765b1d0c7442fe3183de5ba9e8077
+  data.tar.gz: 588b3f6a12b02dc00b096843d61335291d7bf62590337c2f52487df759f4ef05
 SHA512:
-  metadata.gz: 112abfea26523d9592dd208455e70a824937c910f98d910990b2359e0530f8f9fd01348a57b4b067a80ee9a77cf64d36aaf7e4bf704c9cf9c88d99eb324e14f1
-  data.tar.gz: 99fbe2141dee8d99e385b3fe825476366bb56988f7ca508a8ff961d302052b3ab656b07b28cce2513e45012e8a8903aac916f62e31931871fb4a55d4dceb2314
+  metadata.gz: a15cb0b8e9d399a3a37659e7008b3c717337ea14beb6a13ad44e4490cb4faedf59aed7520b9abc265b4c762fdbd7d624a32632886470f915925e183739178496
+  data.tar.gz: fffde7009ac88742aa9694585203de60cee1da4e637821f851eff453f67d9beaa52305049dfe1882373c396b92c933cf84bde9cf8b0fbd4850295a5ea7b3d020

data/lib/hot_cell/filesystem.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module HotCell
+  # What the cell does to trees it owns and did not write: a request's home after the request, the slot
+  # directory at boot, the scratch at boot.
+  module Filesystem
+    # **A mode is the only thing a tool needs to make its own tree unremovable, and a mode on a tree this
+    # uid owns is ours to put back.** `chmod 0500` on a directory a conversion wrote is enough to fail the
+    # recursive delete underneath it, and the delete failing used to be the end of it: one tree per
+    # request stayed on the tmpfs until the container ended. The repair is not a permission the process
+    # gains, it is one it never lost.
+    #
+    # Repair only after a failure, never before, so the common request pays for a walk of its own tree
+    # once rather than twice. `force:` on the chmod because a partial repair that removes most of the tree
+    # is better than none, and the remove that follows is what reports the outcome either way.
+    #
+    # `Dir.exist?` is not the guard, because it follows symlinks and answers false for a dangling one, and
+    # an entry a tool left in a directory's place is exactly what this has to remove.
+    def self.remove_tree(path)
+      return true unless File.exist?(path) || File.symlink?(path)
+
+      FileUtils.remove_entry path
+      true
+    rescue SystemCallError
+      repair_and_remove path
+    end
+
+    def self.repair_and_remove(path)
+      FileUtils.chmod_R 0o700, path, force: true
+      FileUtils.remove_entry path
+      true
+    rescue SystemCallError
+      false
+    end
+    private_class_method :repair_and_remove
+  end
+end

data/lib/hot_cell/log.rb

@@ -38,6 +38,7 @@ module HotCell
       "control.abandoned" => "WARN",
       "control.unanswerable" => "WARN",
       "slot.uncleaned" => "WARN",
+      "scratch.unswept" => "WARN",
       "slot.undiscarded" => "WARN",
     }.freeze
 

data/lib/hot_cell/operation.rb

@@ -201,7 +201,7 @@ module HotCell
       # belongs to the image, which is configured to match the container's CPU quota. Without them
       # `unsetenv_others: true` would hand an exec'd tool the pool the image's bound was meant to take away.
       def tool_environment(overrides)
-        { "HOME" => ENV["HOME"], "PATH" => ENV["PATH"], "LANG" => "C.UTF-8", "LC_ALL" => "C.UTF-8",
+        { "HOME" => ENV["HOME"], "TMPDIR" => ENV["TMPDIR"], "PATH" => ENV["PATH"], "LANG" => "C.UTF-8", "LC_ALL" => "C.UTF-8",
           "OMP_NUM_THREADS" => ENV["OMP_NUM_THREADS"], "OMP_THREAD_LIMIT" => ENV["OMP_THREAD_LIMIT"] }
           .merge(overrides.transform_keys(&:to_s))
           .compact

data/lib/hot_cell/server/version.rb

@@ -2,6 +2,6 @@
 
 module HotCell
   module Server
-    VERSION = "0.3.1"
+    VERSION = "0.4.0"
   end
 end

data/lib/hot_cell/server.rb

@@ -8,6 +8,7 @@ require "hot_cell/limits"
 require "hot_cell/configuration"
 require "hot_cell/registry"
 require "hot_cell/operation"
+require "hot_cell/filesystem"
 require "hot_cell/slot"
 require "hot_cell/log"
 require "hot_cell/timing"

data/lib/hot_cell/slot.rb

@@ -80,7 +80,7 @@ module HotCell
     # rescues anything and a raise stops the cell with every request it holds.
     def remove_home
       return true if home.nil?
-      return false unless remove_tree(home)
+      return false unless Filesystem.remove_tree(home)
 
       self.home = nil
       true
@@ -127,45 +127,18 @@ module HotCell
     # entry a tool left in the slot's place is exactly what this has to remove, and it used to survive the
     # sweep and raise from every later `make_home`.
     def prepare
-      remove_tree directory
+      Filesystem.remove_tree directory
     end
 
     # Unlinks whatever discard_home renamed out of the way. Partial progress is fine: a sweep killed
     # part-way leaves fewer entries for the next one, so this converges rather than repeating.
     def sweep
-      Dir.glob(File.join(directory, "discarded-*")).map { |path| remove_tree(path) }.all?
+      Dir.glob(File.join(directory, "discarded-*")).map { |path| Filesystem.remove_tree(path) }.all?
     rescue SystemCallError
       # The glob itself can fail, because the slot directory is a name a tool can replace — a symlink loop
       # in its place answers ELOOP here rather than for any one entry. This runs from the worker's ensure,
       # where a raise would replace the caller's response with a crash.
       false
     end
-
-    private
-      # **A mode is the only thing a tool needs to make its own tree unremovable, and a mode on a tree this
-      # uid owns is ours to put back.** `chmod 0500` on a directory a conversion wrote is enough to fail the
-      # recursive delete underneath it, and the delete failing used to be the end of it: one tree per
-      # request stayed on the tmpfs until the container ended. The repair is not a permission the process
-      # gains, it is one it never lost.
-      #
-      # Repair only after a failure, never before, so the common request pays for a walk of its own tree
-      # once rather than twice. `force:` on the chmod because a partial repair that removes most of the tree
-      # is better than none, and the remove that follows is what reports the outcome either way.
-      def remove_tree(path)
-        return true unless File.exist?(path) || File.symlink?(path)
-
-        FileUtils.remove_entry path
-        true
-      rescue SystemCallError
-        repair_and_remove path
-      end
-
-      def repair_and_remove(path)
-        FileUtils.chmod_R 0o700, path, force: true
-        FileUtils.remove_entry path
-        true
-      rescue SystemCallError
-        false
-      end
   end
 end

data/lib/hot_cell/supervisor.rb

@@ -4,6 +4,8 @@ require "socket"
 require "fcntl"
 require "fileutils"
 require "tmpdir"
+require "pathname"
+require "etc"
 
 module HotCell
   # Accepts, queues, dispatches, times, kills, reaps, and cleans up. It never evaluates image data.
@@ -139,7 +141,7 @@ module HotCell
     def initialize(directory:, workspace: nil, configuration: HotCell.configuration, log: Log.new,
                    ptrace_scope_path: PTRACE_SCOPE)
       @directory = directory
-      @workspace = workspace || File.join(Dir.tmpdir, "hotcell-workspace")
+      @workspace = workspace || File.join(tmpdir, "hotcell-workspace")
       @configuration = configuration
       @log = log
       @ptrace_scope_path = ptrace_scope_path
@@ -154,6 +156,7 @@ module HotCell
       verify_socket_paths!
       verify_limits!
       verify_ptrace_scope!
+      verify_scratches!
       prepare_directories
       preload
       @work = listen "work.sock"
@@ -983,7 +986,62 @@ module HotCell
         end
       end
 
+      # The scratch is where a killed tool's files outlive it, so the sweep has to be sure of what it is
+      # sweeping. A root that is missing, or is not a directory, has been moved or replaced by something; a
+      # symlink this uid owns anywhere on the path is a redirect a worker could have planted, and a sweep
+      # through it would delete what the link points at; a relative path is whatever the working directory
+      # happens to be, and a `..` after a symlink names one directory to the kernel and another to string
+      # comparison, so the paths have to be already normalized. Symlinks the OS owns stay legal, since `/tmp`
+      # is one on macOS. The socket directory cannot live inside the scratch, because the sweep would have
+      # to skip it, and a skipped name is one a worker can fill.
+      #
+      # What this cannot check is that the root's parent is unwritable, which is what would make the root
+      # immovable. On the accessory `/tmp` is a mount; in development it is a directory under one the
+      # developer owns, and refusing that would refuse every development layout.
+      def verify_scratches!
+        { "workspace" => workspace, "temporary directory" => tmpdir, "socket directory" => directory }.each do |name, path|
+          next if File.absolute_path?(path) && File.expand_path(path) == path
+
+          raise ConfigurationError, "the #{name} #{path} is not an absolute path without `.`, `..` or a trailing slash"
+        end
+
+        scratches.each do |scratch|
+          if (link = owned_symlink_on(scratch))
+            raise ConfigurationError, "#{link} is a symlink the cell's uid owns, on the path to the scratch " \
+                                      "#{scratch}, and a boot sweeps the scratch"
+          end
+          unless File.stat(scratch).directory?
+            raise ConfigurationError, "the scratch #{scratch} is not a directory, and a boot sweeps the scratch"
+          end
+          if directory == scratch || directory.start_with?("#{scratch}/")
+            raise ConfigurationError, "the socket directory #{directory} is inside the scratch #{scratch}, " \
+                                      "which a boot sweeps. Choose a directory outside it."
+          end
+        end
+      rescue Errno::ENOENT => error
+        raise ConfigurationError, "the scratch is missing: #{error.message}"
+      end
+
+      def owned_symlink_on(path)
+        Pathname.new(path).descend.map(&:to_s).find do |ancestor|
+          stat = File.lstat(ancestor)
+          stat.symlink? && stat.uid == Process.uid
+        end
+      end
+
+      # `Dir.tmpdir` with its fallbacks removed: it answers `.` for a `/tmp` its owner cannot write, and a
+      # worker can leave `/tmp` in that state. The sweep is what puts the mode back, so it has to see the
+      # directory `Dir.tmpdir` will answer once it has.
+      def tmpdir
+        ENV.values_at("TMPDIR", "TMP", "TEMP").compact.reject(&:empty?).first || Etc.systmpdir
+      end
+
+      def scratches
+        [ tmpdir, File.dirname(workspace) ].uniq
+      end
+
       def prepare_directories
+        scratches.each { |scratch| sweep_scratch scratch }
         FileUtils.mkdir_p directory
 
         configuration.concurrency.times do |number|
@@ -995,6 +1053,36 @@ module HotCell
         end
       end
 
+      # The scratch outlives the container when it is a host mount, so what a killed tool left at its top is
+      # still there at the next boot, and rebooting the accessory has to be the one command that clears it.
+      # `Dir.tmpdir` is where a tool with no `TMPDIR` writes, and the workspace's parent is the scratch when
+      # `HOTCELL_WORKSPACE` points elsewhere. Only what this uid owns goes, and it goes by the slot's rule:
+      # remove, and put back the modes a tool changed if that fails. `lstat` so a symlink is a link to
+      # unlink and never a tree to walk, and a failure is a line in the log rather than a cell that will
+      # not boot.
+      def sweep_scratch(scratch)
+        repair_root scratch
+        Dir.children(scratch).each do |name|
+          path = File.join(scratch, name)
+          next unless File.lstat(path).uid == Process.uid
+          next if Filesystem.remove_tree(path)
+
+          log.write "scratch.unswept", path: path
+        end
+      rescue SystemCallError => error
+        log.write "scratch.unswept", path: scratch, error: error.class.name, message: error.message
+      end
+
+      # The root's mode is the one a tool can set that the sweep itself trips on: `0300` fails the listing
+      # and `0500` fails every unlink beneath it. It is repaired only when it is wrong, and only when this
+      # uid owns it.
+      def repair_root(scratch)
+        stat = File.stat(scratch)
+        return unless stat.uid == Process.uid && stat.mode & 0o700 != 0o700
+
+        FileUtils.chmod stat.mode | 0o700, scratch
+      end
+
       def preload
         Registry.operations.each { |operation| operation.before_fork.each(&:call) }
       end

data/lib/hot_cell/test_cell.rb

@@ -29,7 +29,7 @@ module HotCell
   class TestCell
     READY = "up"
 
-    attr_reader :name, :directory, :workspace, :log_path
+    attr_reader :name, :directory, :workspace, :tmpdir, :log_path
 
     def self.boot(**options)
       new(**options).start.tap do |cell|
@@ -52,7 +52,10 @@ module HotCell
       @options = options
       @root = Dir.mktmpdir "hotcell-test"
       @directory = File.join(@root, name)
-      @workspace = File.join(@root, "workspace")
+      # Boot empties `Dir.tmpdir` and the workspace's parent of what this uid owns. Inside the cell both are
+      # this directory, so the sweep reaches neither the developer's `/tmp` nor the log and sockets beside it.
+      @tmpdir = File.join(@root, "tmp")
+      @workspace = File.join(@tmpdir, "workspace")
       @log_path = File.join(@root, "cell.log")
     end
 
@@ -75,6 +78,9 @@ module HotCell
 
         HotCell.limits(**@options) unless @options.empty?
 
+        FileUtils.mkdir_p @tmpdir
+        ENV["TMPDIR"] = @tmpdir
+
         supervisor = Supervisor.new(directory: directory, workspace: workspace,
                                     log: Log.new(File.open(log_path, "w")), **@supervisor_options)
         begin

data/lib/hot_cell/test_operations.rb

@@ -3,6 +3,7 @@
 require "digest"
 require "fcntl"
 require "fileutils"
+require "tmpdir"
 
 # Fixture operations, so the whole surface can be exercised in milliseconds, with no tool installed and no
 # container running.
@@ -473,6 +474,21 @@ module HotCell
       end
     end
 
+    # Writes a file where a library writes its scratch, `Dir.tmpdir`, and leaves it there — the way ImageMagick
+    # leaves its pixel cache behind on anything but a clean exit. With `seconds:` it then blocks, so the
+    # deadline kills it mid-request.
+    class Spills < HotCell::Operation
+      operation "test.spills"
+
+      def perform(_inputs, _outputs, seconds: 0)
+        spilled = File.join(Dir.tmpdir, "spill-#{Process.pid}")
+        File.write spilled, "x"
+        sleep seconds
+
+        { spilled: spilled, home: ENV["HOME"] }
+      end
+    end
+
     class Blocking < HotCell::Operation
       operation "test.blocking"
 

data/lib/hot_cell/worker.rb

@@ -116,6 +116,9 @@ module HotCell
           # `run`, which exits the worker — after this ensure had already reported idle `"ok"`, counting a
           # success for a request that never ran.
           ENV["HOME"] = slot.make_home
+          # The home is removed however the request ends, and `/tmp` is swept by nobody. A library's scratch
+          # goes where `TMPDIR` says, so it goes here too — whatever the library, and whatever kills the worker.
+          ENV["TMPDIR"] = ENV["HOME"]
 
           line, received = connection.receive_message
           response = if line.nil?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hotcell-server
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Mike Dalessio
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.3.1
+        version: 0.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.3.1
+        version: 0.4.0
 description: |
   Runs a HotCell container. A supervisor listens on two Unix sockets, forks a worker for each request,
   and enforces a wall clock deadline and resource limits on it. Write the work as a subclass of
@@ -42,6 +42,7 @@ files:
 - lib/hot_cell/configuration.rb
 - lib/hot_cell/control.rb
 - lib/hot_cell/counters.rb
+- lib/hot_cell/filesystem.rb
 - lib/hot_cell/limits.rb
 - lib/hot_cell/log.rb
 - lib/hot_cell/operation.rb
@@ -61,8 +62,8 @@ licenses:
 - MIT
 metadata:
   homepage_uri: https://github.com/basecamp/hotcell
-  source_code_uri: https://github.com/basecamp/hotcell/tree/v0.3.1/hotcell-server
-  changelog_uri: https://github.com/basecamp/hotcell/blob/v0.3.1/CHANGELOG.md
+  source_code_uri: https://github.com/basecamp/hotcell/tree/v0.4.0/hotcell-server
+  changelog_uri: https://github.com/basecamp/hotcell/blob/v0.4.0/CHANGELOG.md
   bug_tracker_uri: https://github.com/basecamp/hotcell/issues
   rubygems_mfa_required: 'true'
 rdoc_options: []