activestorage-hotcell-server 0.0.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d671aeb422ca384fe64dd59d2f53174760c04801f37e75f51488ca5cf34f6fe2
-  data.tar.gz: dae5536d6035f67546136f12ffe1fc52d602bdffc8dd5eeba120fec2313c4cd7
+  metadata.gz: 42399adf3e7993d0df16dd7a13010053ff389ddad17b3c90a6f7705052b3c2e7
+  data.tar.gz: 324c2a21f2e55e7747355f2155542d5ef572164ac53938a1077db3156fbc1c87
 SHA512:
-  metadata.gz: 23612a5bc095201e6131c061067af692af514913a88b13cb04851f72758af0341895c4571a4e195d9dbb58e647321a25eead9b9832da94bf267a6395556fa203
-  data.tar.gz: 8cf8d948c812075b068479e25737a0d43999831f2dea87a018d6d62fe8c1049d96bb27076f88173ed91d4bd4ae1939669d768402ee9e8ce8e5c215c14014d909
+  metadata.gz: f596b308fbdf479792bdb2de90d1fb626ed9207c6194d60a618a1342cca51ae13a7f47a456c15c1ca764f807477070b6d7697f00bc6d4f1dfa7b31788fa37b94
+  data.tar.gz: a6b15cc763f189ce7249bb473f1cd11c24f4d112f8ecad66f97c2e4ad1afb63f845c201dd704e17147a690866dc79b1c49795b61f915719e1c28c91f952236b7

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 37signals, LLC
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,3 @@
+# activestorage-hotcell-server
+
+Part of [HotCell](https://github.com/basecamp/hotcell). See the repository README.

data/lib/active_storage/hot_cell/server/analyzers/image/magick.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "active_storage/hot_cell/server/magick_operation"
+
+module ActiveStorage
+  module HotCell
+    module Server
+      module Analyzers
+        module Image
+          # What `ActiveStorage::Analyzer::ImageAnalyzer::ImageMagick` does, moved out of the application:
+          # width and height from `identify`, with the dimensions swapped when the stored orientation is a
+          # quarter turn, the way the shared image analyzer does it.
+          #
+          # The result is a superset — pages and animation, which the vips analyzer also reports — that the
+          # client analyzer slices to Rails' `{ width, height }`. An input ImageMagick cannot decode exits
+          # `identify` non-zero, which raises MiniMagick::Error, and the cell answers `unreadable`.
+          class Magick < MagickOperation
+            operation "active_storage.analyzers.image.magick"
+
+            limits deadline: 10, memory: 1024 * 1024**2, file_size: 48 * 1024**2, open_files: 64
+
+            # The orientation names identify reports for a quarter turn, matching Rails' rotated_image? on the
+            # ImageMagick path.
+            ROTATED = %w[ LeftTop RightTop RightBottom LeftBottom ].freeze
+
+            def perform(inputs, _outputs)
+              source, = inputs
+              frames = identified(source.path)
+              width, height, orientation = frames.first
+
+              { **dimensions(width, height, orientation), pages: frames.size, animated: frames.size > 1,
+                bytes: source.to_io.stat.size }
+            end
+
+            private
+              # One `identify` run reports every frame's dimensions and orientation, where MiniMagick::Image's
+              # accessors (`valid?`, `width`, `pages`) each spawn an identify of their own — four execs per
+              # analysis, two of them decoding every frame.
+              def identified(path)
+                frames = MiniMagick.identify do |identify|
+                  identify.format "%w %h %[orientation]\n"
+                  identify << path
+                end.lines.map(&:split)
+
+                raise MiniMagick::Invalid, "ImageMagick does not recognise this as an image" if frames.empty?
+
+                frames
+              end
+
+              def dimensions(width, height, orientation)
+                if ROTATED.include?(orientation)
+                  { width: height.to_i, height: width.to_i }
+                else
+                  { width: width.to_i, height: height.to_i }
+                end
+              end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_storage/hot_cell/server/analyzers/image/vips.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "active_storage/hot_cell/server/vips_operation"
+
+module ActiveStorage
+  module HotCell
+    module Server
+      module Analyzers
+        module Image
+          # Metadata and no bytes, which is the request shape with no outputs at all.
+          #
+          # Shipping this is mandatory rather than a nicety. The built-in image analyzers gate `accept?` on
+          # `variant_processor` being `:vips` or `:mini_magick`, so a class value makes them all decline,
+          # `analyzer_class` falls through to `NullAnalyzer`, and the blob is marked analyzed with no dimensions
+          # at all.
+          #
+          # There is a sharper version of the same problem inside the built-in vips analyzer, which this
+          # deliberately does not copy: it rescues every Vips::Error and returns an empty hash, which is then
+          # merged with `analyzed: true`. An undecodable image is recorded as successfully analyzed, forever,
+          # and nothing re-enqueues AnalyzeJob. Here an undecodable input raises Vips::Error, the cell answers
+          # `unreadable`, and the client decides — because only the client knows whether that verdict is safe
+          # to write down.
+          class Vips < VipsOperation
+            operation "active_storage.analyzers.image.vips"
+
+            # Analysis reads a header rather than decoding a whole image, so it gets far less room than a
+            # transform.
+            limits deadline: 10, memory: 1024 * 1024**2, file_size: 48 * 1024**2, open_files: 64
+
+            def perform(inputs, _outputs)
+              source, = inputs
+              image = ::Vips::Image.new_from_file(source.fd_path, access: :sequential)
+
+              { **dimensions_of(image), **frames_of(image),
+                bytes: source.to_io.stat.size, tracked_mem_highwater: vips_highwater }
+            end
+
+            private
+              # `pages` is what tells a caller whether asking for `loader: { n: -1 }` would mean anything, and
+              # it is one of the things a custom analyzer produces that Rails' does not.
+              def frames_of(image)
+                pages = image.get("n-pages").to_i
+                { pages: pages, animated: pages > 1 }
+              rescue ::Vips::Error
+                { pages: 1, animated: false }
+              end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_storage/hot_cell/server/analyzers/media/ffprobe.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require "json"
+require "active_storage/hot_cell/server/tool_operation"
+
+module ActiveStorage
+  module HotCell
+    module Server
+      module Analyzers
+        module Media
+          # What Rails' video and audio analyzers do, moved out of the application: ffprobe, and the numbers it
+          # reports shaped exactly as `Analyzer::VideoAnalyzer` and `Analyzer::AudioAnalyzer` shape them. One
+          # operation serves both client analyzers, because ffprobe reports video and audio streams in one pass.
+          #
+          # **This reads its tool's stdout, and that is a judgement, so here it is.**
+          #
+          # ffprobe parses the media in an exec'd child that dies at the end of the call. What comes back into this
+          # worker is JSON on a bounded buffer, parsed by `JSON.parse` — not a media decoder, and not a parser with a
+          # history of memory-safety bugs. The question that decides whether recycling a worker is safe is whether a
+          # malicious input can execute in this process, and through ffprobe's JSON it cannot. Reading a tool's
+          # *output file* with an in-process media library is in-process decoding; parsing its structured stdout with
+          # the standard library is not.
+          #
+          # The result is a superset of what either analyzer writes, and the client analyzers slice it to Rails'
+          # exact keys — the same split the image analyzer uses. Two things here are deliberately not Rails. Every
+          # number is coerced tolerantly rather than with `Integer()`/`Float()` that raise, because ffprobe's output
+          # is attacker-controlled where Rails' is trusted; a value that is not a clean number is dropped rather than
+          # crashing the analysis. And `tags` — title, artist, arbitrary bytes that need not be valid UTF-8 — are
+          # never returned, where Rails writes them straight into the database.
+          class Ffprobe < ToolOperation
+            operation "active_storage.analyzers.media.ffprobe"
+
+            limits deadline: 30, memory: 1024 * 1024**2, file_size: 48 * 1024**2, open_files: 128
+
+            ROTATIONS = [ 90, 270, -90, -270 ].freeze
+
+            # `probe_arguments` is `config.active_storage.ffprobe_arguments`, split, and it goes where Rails
+            # puts it: before the input path, which is where an input option such as `-codec_whitelist` has
+            # to be to take effect.
+            def perform(inputs, _outputs, probe_arguments: [])
+              source, = inputs
+              probed = JSON.parse(run!("ffprobe", "-v", "quiet", "-print_format", "json",
+                                       "-show_format", "-show_streams",
+                                       *arguments!(:probe_arguments, probe_arguments), source.fd_path,
+                                       pass: [ source.to_io ]).out)
+
+              video = stream_of(probed, "video")
+              audio = stream_of(probed, "audio")
+
+              { duration: duration(probed, video, audio), bytes: source.to_io.stat.size,
+                video: !video.nil?, audio: !audio.nil? }
+                .merge(video ? video_metadata(video) : {})
+                .merge(audio ? audio_metadata(audio) : {})
+                .compact
+            rescue JSON::ParserError => error
+              raise UnreadableDocument, "ffprobe said something that is not JSON: #{error.message[0, 120]}"
+            end
+
+            private
+              def stream_of(probed, type)
+                Array(probed["streams"]).find { |stream| stream["codec_type"] == type }
+              end
+
+              def duration(probed, video, audio)
+                source = if video
+                  video["duration"] || probed.dig("format", "duration")
+                elsif audio
+                  audio["duration"]
+                end
+
+                float(source)
+              end
+
+              # Rails' width and height: the encoded width, and a height recomputed from the display aspect ratio
+              # so an anamorphic stream reports its displayed shape rather than its stored one — width-preserving,
+              # the way VideoAnalyzer does it. Swapped when the stream is stored rotated a quarter turn.
+              def video_metadata(stream)
+                encoded_width = float(stream["width"])
+                encoded_height = float(stream["height"])
+                ratio = display_aspect_ratio(stream)
+                displayed_height = computed_height(encoded_width, ratio) || encoded_height
+                angle = angle(stream)
+                rotated = ROTATIONS.include?(angle)
+
+                { width: rotated ? displayed_height : encoded_width,
+                  height: rotated ? encoded_width : displayed_height,
+                  angle: angle, display_aspect_ratio: ratio,
+                  video_codec: SCRUB[stream["codec_name"]] }
+              end
+
+              def audio_metadata(stream)
+                { sample_rate: integer(stream["sample_rate"]), bit_rate: integer(stream["bit_rate"]),
+                  audio_codec: SCRUB[stream["codec_name"]] }
+              end
+
+              def computed_height(encoded_width, ratio)
+                return nil unless encoded_width && ratio
+
+                encoded_width * (ratio.last.to_f / ratio.first)
+              end
+
+              def display_aspect_ratio(stream)
+                descriptor = stream["display_aspect_ratio"].to_s
+                numerator, denominator = descriptor.split(":", 2).map { |term| integer(term) }
+                [ numerator, denominator ] if numerator&.positive? && denominator&.positive?
+              end
+
+              # The angle Rails reports: a rotate tag first, then a Display Matrix side-data entry. Anything that
+              # is not a clean integer is nothing, because this rides untrusted ffprobe output.
+              def angle(stream)
+                tags = stream["tags"] || {}
+                return integer(tags["rotate"]) if tags["rotate"]
+
+                matrix = Array(stream["side_data_list"]).find { |data| data["side_data_type"] == "Display Matrix" }
+                integer(matrix["rotation"]) if matrix
+              end
+
+              # Numbers, and codec names matched against a conservative pattern. A tag or a codec that is not one
+              # of these is dropped rather than carried into a database as arbitrary bytes.
+              CODEC = /\A[a-z0-9_]{1,32}\z/
+              SCRUB = ->(value) { value.to_s.match?(CODEC) ? value.to_s : nil }
+
+              def integer(value)
+                number = float(value)
+                number&.to_i
+              end
+
+              def float(value)
+                return nil if value.nil?
+
+                parsed = Float(value, exception: false)
+                parsed if parsed&.finite?
+              end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_storage/hot_cell/server/magick_operation.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "active_storage/hot_cell/server/operation"
+
+# Loaded here rather than lazily in a worker, so mini_magick resolves the ImageMagick binary once at boot
+# rather than shelling out to find it on the first request under a tight limit. mini_magick shells out for
+# every operation, so unlike libvips there is no thread pool that fork would break — but keeping the two
+# toolchains' loads in the same place is what makes a single cell carrying both legible.
+require "image_processing/mini_magick"
+
+# Invariant 9: a tool sees only the environment its operation wrote for it. `Operation#run_tool` holds it
+# with `unsetenv_others: true`, and these operations do not use it — mini_magick spawns `magick` itself.
+# `MiniMagick.restricted_env` is `false` by default, and with it false mini_magick calls
+# `Open3.popen3({}, *command, unsetenv_others: false)`, so `magick` inherited this worker's whole
+# environment. `bin/conformance` did not catch it, because its environment check drives an operation that
+# goes through `run_tool`.
+#
+# `cli_env` names the locale for the reason `tool_environment` does: a tool's output must not shift under
+# it. mini_magick passes `HOME`, `PATH` and `LANG` and does not pass `LC_ALL`, so both go here.
+#
+# Set at require time rather than in `before_worker_boot`, so that the binary lookup this require performs
+# is covered too. Note what this is: mini_magick filters the environment it was given, where `run_tool`
+# writes a fresh one. Driving `magick` directly would make it ours, and that is the separate enhancement
+# this class already names.
+MiniMagick.restricted_env = true
+MiniMagick.cli_env = { "LANG" => "C.UTF-8", "LC_ALL" => "C.UTF-8" }
+
+module ActiveStorage
+  module HotCell
+    module Server
+      # Everything that transforms or analyses an image with ImageMagick, through mini_magick.
+      #
+      # ImageMagick runs as an exec'd `magick` process that dies at the end of the call, so a decompression
+      # bomb executes in a child rather than in this worker. What this does touch in-process is mini_magick's
+      # own output — dimensions from `identify`, an exit status — which is a number and a string, not a
+      # decoder. The distinction is the same one probe_media draws for ffprobe.
+      #
+      # Unlike the vips operations, the input is staged onto scratch: mini_magick spawns its own `magick` and
+      # does not inherit this worker's descriptors, so an input cannot be handed to it as /dev/fd. That
+      # returns the file_size ceiling to this path — an input larger than the operation's file_size dies
+      # being staged — which the vips operations shed. Removing it means driving `magick` directly rather
+      # than through mini_magick, and is a separate enhancement.
+      class MagickOperation < Operation
+        abstract_operation
+
+        # MiniMagick::Error is how mini_magick reports a `magick` that exited non-zero — the common shape of an
+        # input it cannot decode. MiniMagick::Invalid is an input `identify` rejects outright. Both are the
+        # input's fault rather than the operation's.
+        unreadable MiniMagick::Error, MiniMagick::Invalid
+
+        # **Accepted risk.** A tool's output is not bounded on this path. `Operation#run_tool` caps what it
+        # reads at 64KB and drops the rest as it arrives, because an input that makes a tool print gigabytes
+        # of diagnostics costs this worker gigabytes of address space, takes RLIMIT_DATA with it, and arrives
+        # as a `memory` verdict — which is permanent, for a document whose only crime was being noisy.
+        # mini_magick reads both streams to EOF in a thread apiece and has no setting that bounds either;
+        # `graphicsmagick`, `cli_prefix`, `cli_env`, `restricted_env`, `timeout`, `logger`, `tmpdir`,
+        # `errors` and `warnings` are the whole list. The premise is that patching the library is worse than
+        # carrying this, and that the operations move off it: driving `magick` directly through `run_tool`
+        # bounds the output, returns the environment to us, and sheds the input staging, which is
+        # basecamp/hotcell#7.
+      end
+    end
+  end
+end

data/lib/active_storage/hot_cell/server/operation.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "hot_cell/server"
+
+module ActiveStorage
+  # Despite the namespace, nothing here loads Active Storage. The name says which consumer these operations
+  # serve, not what they link against, and the gem stays small because a smaller graph is a smaller thing to
+  # audit — not because a cell polices what runs inside it. It does not: the container is the control, and what
+  # an operation chooses to require is its own business.
+  module HotCell
+    # Everything the cell side defines lives under this, and everything the application side defines lives under
+    # ActiveStorage::HotCell::Client. Not a tidying convention: the two gems are never both loaded in production,
+    # and a cell is forked from a process that may well have loaded the client — after which a shared name is a
+    # superclass mismatch while the cell boots. Two namespaces make that impossible rather than avoided.
+    module Server
+      # What every operation in this gem shares, whichever toolchain performs it.
+      class Operation < ::HotCell::Operation
+        abstract_operation
+
+        # A lookup, not a gate: which format a caller may ask for is decided by the toolchain's own build,
+        # the way Rails decides it. This only names the content type in a result, and a format it has never
+        # heard of gets none.
+        CONTENT_TYPES = {
+          "png"  => "image/png",
+          "jpg"  => "image/jpeg",
+          "jpeg" => "image/jpeg",
+          "webp" => "image/webp",
+          "gif"  => "image/gif",
+          "avif" => "image/avif",
+          "tiff" => "image/tiff",
+        }.freeze
+
+        private
+          # A caller breaking the protocol is a caller bug, not a bad document. Raising MessageError is what
+          # makes the cell answer `invalid`, which is permanent and which the client raises rather than
+          # turning into a placeholder.
+          def refuse!(message)
+            raise ::HotCell::MessageError, message
+          end
+      end
+    end
+  end
+end

data/lib/active_storage/hot_cell/server/previewers/pdf/mutool.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "active_storage/hot_cell/server/previewers/pdf"
+
+module ActiveStorage
+  module HotCell
+    module Server
+      module Previewers
+        class Pdf
+          # What `ActiveStorage::Previewer::MuPDFPreviewer` does, moved out of the application.
+          #
+          # Rails runs `mutool draw -F png -o - <file> 1` and streams the result through the web process. This
+          # writes to the worker's own scratch instead of to stdout.
+          class Mutool < Pdf
+            operation "active_storage.previewers.pdf.mutool"
+
+            private
+              # The input is read through its descriptor, but the output is staged: mutool unlinks its output
+              # path before writing, and /dev/fd cannot be unlinked, so a passed output descriptor fails with
+              # "Operation not permitted". The staged PNG is copied out by Output#post. Streaming mutool's
+              # stdout to the descriptor — what Rails does with `-o -` — would remove the copy, and is a
+              # separate change.
+              def render(source, destination, page:, resolution:)
+                run! "mutool", "draw", "-F", "png", "-r", resolution.to_s, "-o", destination.path,
+                     source.fd_path, page.to_s, pass: [ source.to_io ]
+              end
+
+              def tool
+                "mutool"
+              end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_storage/hot_cell/server/previewers/pdf/poppler.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "active_storage/hot_cell/server/previewers/pdf"
+
+module ActiveStorage
+  module HotCell
+    module Server
+      module Previewers
+        class Pdf
+          # What `ActiveStorage::Previewer::PopplerPDFPreviewer` does, moved out of the application: the Poppler
+          # sibling of the mutool preview, for an image whose Rails default previewer chain reaches pdftoppm
+          # first.
+          #
+          # Rails runs `pdftoppm -singlefile -cropbox -r 72 -png <file>` and streams the result through the web
+          # process. This reads the input through its descriptor and writes to the worker's own scratch.
+          class Poppler < Pdf
+            operation "active_storage.previewers.pdf.poppler"
+
+            private
+              # pdftoppm appends `.png` to its output root, so the frame lands beside the scratch name and is
+              # adopted into place; Output#post makes the one copy out through the caller's descriptor.
+              def render(source, destination, page:, resolution:)
+                run! "pdftoppm", "-png", "-singlefile", "-cropbox", "-r", resolution.to_s,
+                     "-f", page.to_s, "-l", page.to_s, source.fd_path, destination.path,
+                     pass: [ source.to_io ]
+
+                destination.adopt destination.path(extension: "png")
+              end
+
+              def tool
+                "pdftoppm"
+              end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_storage/hot_cell/server/previewers/pdf.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "active_storage/hot_cell/server/tool_operation"
+
+module ActiveStorage
+  module HotCell
+    module Server
+      module Previewers
+        # What the two PDF previewers share: one bounded page, rendered to PNG on the worker's own scratch, so
+        # a pathological page cannot be answered by buffering an unbounded PNG in memory — the file is bounded
+        # by the cell's `file_size` limit and the kernel enforces it. A subclass supplies the tool run.
+        #
+        # The result carries no dimensions, and that is what Rails does rather than a concession to make here. A
+        # previewer yields `io:`, `filename:` and `content_type:` and nothing else; `Preview#process` attaches that
+        # as a new blob, and the dimensions come later from analyzing *that* blob like any other. So a drop-in
+        # replacement returns no dimensions either.
+        #
+        # It is also what keeps the `:subprocess` claim true. Reading the produced PNG here would mean parsing
+        # bytes the tool just made out of a hostile PDF, in this worker, which is exactly what turns a
+        # subprocess operation into an in-process one.
+        class Pdf < ToolOperation
+          abstract_operation
+
+          limits deadline: 30, memory: 1024 * 1024**2, file_size: 48 * 1024**2, open_files: 128
+
+          MAX_PAGE = 10_000
+          MAX_RESOLUTION = 600
+
+          def perform(inputs, outputs, page: 1, resolution: 72)
+            source, = inputs
+            destination, = outputs
+
+            page = positive_integer!(:page, page, MAX_PAGE)
+            resolution = positive_integer!(:resolution, resolution, MAX_RESOLUTION)
+
+            render source, destination, page: page, resolution: resolution
+
+            staged = File.exist?(destination.path) ? File.size(destination.path) : 0
+
+            { format: "png", content_type: "image/png", page: page, resolution: resolution,
+              bytes: produced!(staged, tool) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_storage/hot_cell/server/previewers/video/ffmpeg.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require "active_storage/hot_cell/server/tool_operation"
+
+module ActiveStorage
+  module HotCell
+    module Server
+      module Previewers
+        module Video
+          # What `ActiveStorage::Previewer::VideoPreviewer` does, moved out of the application.
+          #
+          # Rails runs `ffmpeg <input args> -i <file> <output args> -` and lets an application set both
+          # argument lists as shell strings it splits with Shellwords. This honours the input half:
+          # `video_preview_input_arguments` arrives split, and it goes before `-i` exactly as it does in
+          # Rails, because that is where an input option has to be. The output half is fixed here — see
+          # RELEVANT_FRAME — and the only other thing a caller says is how many seconds in to seek.
+          class Ffmpeg < ToolOperation
+            operation "active_storage.previewers.video.ffmpeg"
+
+            # Video is the reason a cell exists as a separate accessory. A preview measured in minutes and a
+            # thumbnail measured in milliseconds must not share a concurrency limit, so this is sized to be
+            # given a cell of its own rather than dropped in beside the image operations.
+            limits deadline: 120, memory: 1536 * 1024**2, file_size: 128 * 1024**2, open_files: 128
+
+            MAX_SEEK = 86_400
+
+            # Rails' own frame selection, verbatim from the 7.0 defaults: frame 0, keyframes, and scene changes
+            # over 0.015 are selected, and the loop/trim pair yields the second of those with the first as the
+            # fallback — so a video that opens on black previews as its first scene rather than as the black.
+            # This is the output half of `video_preview_arguments`, and it stays fixed: the input half is
+            # what an application needs to shape, and that travels separately as `input_arguments`.
+            RELEVANT_FRAME = 'select=eq(n\,0)+eq(key\,1)+gt(scene\,0.015),loop=loop=-1:size=2,trim=start_frame=1'
+
+            def perform(inputs, outputs, seek: 0, input_arguments: [])
+              source, = inputs
+              destination, = outputs
+
+              seek = seek!(seek)
+              input_arguments = arguments!(:input_arguments, input_arguments)
+
+              # JPEG rather than PNG, because Rails' own video previewer yields image/jpeg and this is meant to
+              # drop into its place. A preview that changed the attached blob's content type would not be a
+              # replacement.
+              #
+              # -ss before -i seeks by keyframe, which is orders of magnitude cheaper on a long file than
+              # decoding up to the point. -nostdin because ffmpeg reads the terminal otherwise, and a worker has
+              # no terminal. The application's input arguments go directly before -i, so they cannot drift
+              # past it as the fixed flags around them change.
+              #
+              # Both descriptors go to ffmpeg: it reads /dev/fd for the source and writes /dev/fd for the frame,
+              # so neither the input nor the output touches scratch. Writing the descriptor directly leaves
+              # partial bytes in the caller's file on a mid-write failure where a staged output would have left
+              # it empty — harmless here, because run! turns a non-zero exit into a refusal and the previewer
+              # discards its output on any failure, but it is a real property of the direct write.
+              run! "ffmpeg", "-nostdin", "-loglevel", "error", "-ss", seek.to_s,
+                   *input_arguments, "-i", source.fd_path,
+                   "-vf", RELEVANT_FRAME,
+                   "-frames:v", "1", "-f", "image2", "-c:v", "mjpeg", "-y", destination.fd_path,
+                   pass: [ source.to_io, destination.to_io ]
+
+              { format: "jpg", content_type: "image/jpeg", seek: seek,
+                bytes: produced!(destination.to_io.stat.size, "ffmpeg") }
+            end
+
+            private
+              def seek!(seek)
+                return seek if seek.is_a?(Numeric) && !seek.negative? && seek <= MAX_SEEK
+
+                refuse! "seek #{seek.inspect} must be a number of seconds between 0 and #{MAX_SEEK}"
+              end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_storage/hot_cell/server/tool_operation.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "active_storage/hot_cell/server/operation"
+
+module ActiveStorage
+  module HotCell
+    module Server
+      # Everything that hands the untrusted bytes to an exec'd tool rather than parsing them here.
+      #
+      # A malicious input executes inside a child that dies at the end of the conversion, and this worker only
+      # copies bytes, spawns, and reads an exit status. That is worth knowing when reading these operations,
+      # and it is worth being careful about when changing them: reading a tool's output with an
+      # in-process library — to put dimensions in the result, say — or parsing its stdout brings bytes a
+      # hostile input produced back into this worker. Neither happens here.
+      class ToolOperation < Operation
+        abstract_operation
+
+        # A tool that exits non-zero on a document it cannot read is the commonest failure on this path, and
+        # it is the input's fault rather than the operation's. No `unreadable` declaration is needed:
+        # UnreadableInput is always in the rescue list, and this descends from it.
+        class UnreadableDocument < ::HotCell::UnreadableInput; end
+
+        private
+          # Runs the tool with unsetenv_others and a written environment, and turns a non-zero exit into
+          # `unreadable`. The tool's own stderr is the only useful diagnostic, and it is attacker-influenced,
+          # so it is capped and scrubbed on its way onto the wire like any other error message.
+          def run!(*command, pass: [])
+            result = run_tool(*command, pass: pass)
+            return result if result.ok?
+
+            raise UnreadableDocument, "#{command.first} exited #{result.status.exitstatus}: " \
+                                      "#{result.err.to_s.strip[0, 200]}"
+          end
+
+          # An empty output means the tool said it succeeded and produced nothing, which the client would
+          # otherwise have to catch as zero bytes on a successful response. The caller measures the output
+          # wherever it landed — the descriptor's own file for a tool that wrote it directly, or the staged
+          # scratch path for one that could not.
+          def produced!(bytes, command)
+            raise UnreadableDocument, "#{command} wrote nothing" if bytes.zero?
+
+            bytes
+          end
+
+          def positive_integer!(name, value, maximum)
+            return value if value.is_a?(Integer) && value.positive? && value <= maximum
+
+            refuse! "#{name} #{value.inspect} must be an integer between 1 and #{maximum}"
+          end
+
+          # Extra command-line arguments an application configures — `config.active_storage.ffprobe_arguments`
+          # and its siblings — spliced into the tool's argv at the position that setting names. The client
+          # splits the shell string, so what arrives is already argv, and this checks only that it is: what
+          # the flags mean is the application's decision, exactly as it is when Rails runs the tool itself.
+          #
+          # **Accepted risk.** These are tool options and nothing here restricts which. `-o` makes ffprobe
+          # write a file, `-dump_attachment` makes ffmpeg extract one, and no shell is involved in either.
+          # The premise is that the payload comes from the trusted side: an application's own configuration
+          # crosses a socket only that application can reach, and Rails hands the same strings to the same
+          # tools when it runs them itself. An allowlist here would be a second, worse copy of a decision the
+          # application already made. What would change the premise is the work socket becoming reachable by
+          # something other than the application.
+          def arguments!(name, value)
+            return value if value.is_a?(Array) && value.all?(String)
+
+            refuse! "#{name} must be an array of strings, and this is #{value.inspect[0, 80]}"
+          end
+      end
+    end
+  end
+end

data/lib/active_storage/hot_cell/server/transformers/image/magick.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "active_storage/hot_cell/server/magick_operation"
+require "active_storage/hot_cell/server/transforming"
+
+module ActiveStorage
+  module HotCell
+    module Server
+      module Transformers
+        module Image
+          # What `ActiveStorage::Transformers::ImageMagick` does, moved out of the application: the same
+          # `source(file).loader(page: 0).convert(format).apply(operations)` pipeline, run through
+          # ImageProcessing::MiniMagick.
+          #
+          # The transformation allowlist Rails' ImageMagick transformer enforces —
+          # `supported_image_processing_methods` and the argument blocklist — runs on the client, where an
+          # application's Rails configuration applies today, and is deliberately not repeated here.
+          # ImageProcessing still refuses a name that is neither one of its operations nor a MiniMagick method,
+          # so `:system` and friends cannot reach `magick`.
+          class Magick < MagickOperation
+            include Transforming
+
+            operation "active_storage.transformers.image.magick"
+
+            limits deadline: 30, memory: 1280 * 1024**2, file_size: 48 * 1024**2, open_files: 256
+
+            private
+              def processor
+                ImageProcessing::MiniMagick
+              end
+
+              # The staged path rather than /dev/fd; MagickOperation's comment says why.
+              def source_path(source)
+                source.path
+              end
+
+              def describe(path, format)
+                { format: format, content_type: CONTENT_TYPES[format.downcase], bytes: File.size(path) }.compact
+              end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_storage/hot_cell/server/transformers/image/vips.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "active_storage/hot_cell/server/vips_operation"
+require "active_storage/hot_cell/server/transforming"
+
+module ActiveStorage
+  module HotCell
+    module Server
+      module Transformers
+        module Image
+          # What `ActiveStorage::Transformers::ImageProcessingTransformer` does, moved out of the application,
+          # and deliberately nothing more: the transformations and the format reach ImageProcessing exactly as
+          # Rails hands them over.
+          #
+          # There is no allowlist here, and that mirrors Rails: the vips path validates exactly one thing —
+          # `combine_options` is refused — and `ActiveStorage.supported_image_processing_methods` is enforced by
+          # the ImageMagick transformer alone. ImageProcessing itself refuses a name that is not one of its own
+          # operations or a `Vips::Image` method, which rules out `:system` and friends. Bounding the operation
+          # set and the keys inside `loader`/`saver` is a planned, separate deliverable — see the README — and
+          # until then the cell's limits are the bound: RLIMIT_DATA, RLIMIT_FSIZE and the deadline still apply,
+          # so a pipeline libvips has been told not to bound costs the caller a killed worker.
+          class Vips < VipsOperation
+            include Transforming
+
+            operation "active_storage.transformers.image.vips"
+
+            limits deadline: 30, memory: 1280 * 1024**2, file_size: 48 * 1024**2, open_files: 256
+
+            private
+              def processor
+                ImageProcessing::Vips
+              end
+
+              def source_path(source)
+                source.fd_path
+              end
+
+              # Read back what was just written. Several callers need it: an analyzer returns metadata and no
+              # bytes at all, and a thumbnailer needs the output's own dimensions rather than the ones it asked
+              # for.
+              #
+              # This parses bytes libvips produced from a hostile input, in this worker. That is already true of
+              # everything above: this operation parses hostile bytes in the worker, and reading its own output
+              # is one more place it does so.
+              def describe(path, format)
+                image = ::Vips::Image.new_from_file(path)
+
+                { format: format, content_type: CONTENT_TYPES[format.downcase], bytes: File.size(path),
+                  tracked_mem_highwater: vips_highwater, **dimensions_of(image) }.compact
+              end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_storage/hot_cell/server/transforming.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require "active_storage/hot_cell/server/operation"
+
+module ActiveStorage
+  module HotCell
+    module Server
+      # The transform shared by the two toolchains. An includer names its ImageProcessing backend with
+      # `processor`, how the source reaches it with `source_path`, and what it reports with `describe`.
+      module Transforming
+        def perform(inputs, outputs, format:, operations: {})
+          source, = inputs
+          destination, = outputs
+
+          format = format.to_s
+
+          # ImageProcessing chooses its saver from the destination's extension, and a scratch path has none,
+          # so encode to a suffixed sibling on the slot's own scratch and adopt it into place. That is one
+          # rename rather than the copy-out-of-Dir.tmpdir that a destination-less `call` would do, and it
+          # keeps ImageProcessing's own saver — quality, strip, format defaults — rather than reaching past
+          # it to the library, which cannot reproduce those without restating them. Output#post makes the
+          # one remaining copy, out through the caller's descriptor.
+          encoded = destination.path(extension: format)
+          pipeline(source_path(source), format, operations).call(destination: encoded)
+          destination.adopt encoded
+
+          describe destination.path, format
+        end
+
+        private
+          # `loader(page: 0)` first is what Rails does, and it is what stops a multi-page TIFF or a
+          # hundred-frame GIF being decoded in full to produce one thumbnail. A caller's own `loader` arrives
+          # through `apply` and merges over it, which is also what Rails does — asking for every frame is
+          # `loader: { n: -1 }`.
+          def pipeline(path, format, operations)
+            processor
+              .source(path)
+              .loader(page: 0)
+              .convert(format)
+              .apply(operations_for(operations))
+          end
+
+          # What Rails validates, and no more. `combine_options` is refused because it can never be one
+          # pipeline, and a blank argument means "skip this operation" — Rails filters on `present?`, and the
+          # test below is that semantic reimplemented, because the cell does not load Active Support.
+          #
+          # **Accepted risk.** Any other name reaches ImageProcessing, which dispatches it to the backend.
+          # `ImageProcessing.unsafe_method?` stops the Ruby core methods, so `send` and `system` cannot be
+          # reached, and it does not stop a genuine one: `write_to_file` and `dzsave` are real libvips
+          # operations that write where they are told. The premise is that the transformations come from the
+          # trusted side, and that this is no more permissive than stock Rails, whose own Vips transformer
+          # refuses `combine_options` and nothing else. The ImageMagick allowlist Rails added after
+          # CVE-2022-21831 runs on the client, where the application's configuration for it applies — see
+          # `Transformers::Image::Magick` on that side, which calls `operations` for that effect.
+          def operations_for(declared)
+            refuse! "operations must be an object, and this is a #{declared.class}" unless declared.is_a?(Hash)
+
+            declared.filter_map do |name, argument|
+              if name.to_s == "combine_options"
+                refuse! "combine_options is not supported, because it cannot generate a single command"
+              end
+
+              [ name, argument ] unless blank?(argument)
+            end
+          end
+
+          def blank?(argument)
+            return true if argument.nil? || argument == false
+            return argument.match?(/\A[[:space:]]*\z/) if argument.is_a?(String)
+
+            argument.respond_to?(:empty?) && argument.empty?
+          end
+      end
+    end
+  end
+end

data/lib/active_storage/hot_cell/server/version.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module ActiveStorage
+  module HotCell
+    module Server
+      VERSION = "0.2.0"
+    end
+  end
+end

data/lib/active_storage/hot_cell/server/vips_operation.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require "active_storage/hot_cell/server/operation"
+
+# Loaded here, in the supervisor, rather than lazily in a worker.
+#
+# Two reasons, and the second is the one that bites. It forces libvips' loader plugins to be dlopened now: left
+# to load lazily in a worker, they load *after* that worker's limits are on, and a tight memory limit makes
+# dlopen fail with only a VIPS-WARNING — after which the process runs on with HEIC and AVIF quietly missing. A
+# resource-limit breach becomes `unreadable` for an entire format family, which is the worst confusion this
+# error taxonomy can produce.
+#
+# It is also what makes `unreadable Vips::Error` below expressible at all.
+require "image_processing/vips"
+
+module ActiveStorage
+  module HotCell
+    module Server
+      # Everything that parses an image with libvips, which happens in the worker's own address space.
+      class VipsOperation < Operation
+        abstract_operation
+
+        # Vips::Error is how libvips reports a file it cannot decode, which is common rather than exceptional: it
+        # covers truncated uploads, formats this build was not compiled with, and formats deliberately refused by
+        # block_untrusted. All of those are the input's fault and none is the operation's.
+        unreadable Vips::Error
+
+        # Requires and configures. It must never evaluate an image, and the reason is mechanical: libvips starts
+        # its thread pool on the first evaluation, that pool does not survive fork, and the child then waits on a
+        # pool with no threads.
+        #
+        # Measured on this design's own machine, with libvips 8.18: after the require and the block below, the
+        # process has three threads and every forked child converts. One `Vips::Image.black(1,1).avg` takes it to
+        # five, and from then on every forked child blocks forever in futex_do_wait. Not the first child — every
+        # child. The suite holds this.
+        before_fork do
+          unless Vips.respond_to?(:block_untrusted)
+            raise ::HotCell::ConfigurationError,
+                  "libvips' unfuzzed operations are not safe on untrusted content and this build cannot disable " \
+                  "them. That needs libvips 8.13 or later and ruby-vips 2.2.1 or later."
+          end
+        end
+
+        # There is no `Vips.block_untrusted true` here, and that is a decision rather than an omission.
+        #
+        # The gemspec pins image_processing to 2.0 or later, which blocks libvips' unfuzzed loaders as it loads.
+        # It skips that call when `VIPS_BLOCK_UNTRUSTED` is in the environment, and measuring rather than reading
+        # is what settled this: libvips honours that variable itself, so the loaders are blocked with the variable
+        # unset, set, or set to the empty string. A call here would be a fourth road to a place already reached
+        # three ways.
+        #
+        # What none of them covers is somebody calling `Vips.block_untrusted false` afterwards — and a call at
+        # worker boot would not cover it either, since an operation could do it inside `perform`.
+        # blocked_loaders_test.rb holds the property, which is the thing worth holding.
+        #
+        # The operation cache is set to nothing on purpose. Above `max_requests_per_worker: 1` it would span requests inside one
+        # worker, which is a place one request's image data can sit while the next one runs.
+        before_worker_boot do
+          Vips.concurrency_set Integer(ENV.fetch("VIPS_CONCURRENCY", "2"))
+          Vips.cache_set_max 0
+          Vips.cache_set_max_mem 0
+        end
+
+        private
+          # EXIF says the pixels are stored rotated, so the dimensions a caller cares about are swapped. Rails'
+          # own analyzer does the same thing for the same orientations.
+          SWAPPED_ORIENTATIONS = (5..8)
+
+          def dimensions_of(image)
+            if SWAPPED_ORIENTATIONS.cover?(orientation_of(image))
+              { width: image.height, height: image.width }
+            else
+              { width: image.width, height: image.height }
+            end
+          end
+
+          def orientation_of(image)
+            image.get("exif-ifd0-Orientation").to_i
+          rescue ::Vips::Error
+            0
+          end
+
+          # The only number that says whether a cell's `memory` limit is sized right: RSS understates the charge
+          # and VmSize overstates it by more than twice.
+          def vips_highwater
+            ::Vips.tracked_mem_highwater
+          end
+      end
+    end
+  end
+end

data/lib/active_storage/hot_cell/server.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "active_storage/hot_cell/server/version"
+
+require "active_storage/hot_cell/server/operation"
+require "active_storage/hot_cell/server/transforming"
+
+require "active_storage/hot_cell/server/vips_operation"
+require "active_storage/hot_cell/server/transformers/image/vips"
+require "active_storage/hot_cell/server/analyzers/image/vips"
+
+require "active_storage/hot_cell/server/magick_operation"
+require "active_storage/hot_cell/server/transformers/image/magick"
+require "active_storage/hot_cell/server/analyzers/image/magick"
+
+require "active_storage/hot_cell/server/tool_operation"
+require "active_storage/hot_cell/server/previewers/pdf"
+require "active_storage/hot_cell/server/previewers/pdf/mutool"
+require "active_storage/hot_cell/server/previewers/pdf/poppler"
+require "active_storage/hot_cell/server/previewers/video/ffmpeg"
+require "active_storage/hot_cell/server/analyzers/media/ffprobe"

data/lib/activestorage-hotcell-server.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+# Bundler auto-requires this gem as "activestorage-hotcell-server", then as "activestorage/hotcell/server".
+# It uses neither path. Without this file, naming it in a Gemfile silently loads nothing.
+require "active_storage/hot_cell/server"

metadata

@@ -1,24 +1,108 @@
 --- !ruby/object:Gem::Specification
 name: activestorage-hotcell-server
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Mike Dalessio
 bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
-dependencies: []
-description: To be released soon, secure sidecar for Rails with Active Storage extensions
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: hotcell-server
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: image_processing
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: mini_magick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: ruby-vips
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.2'
+description: |
+  The HotCell operations behind activestorage-hotcell-client: image transformation with libvips or
+  ImageMagick, image and media analysis, PDF previews with mutool or poppler, and video previews with
+  ffmpeg. Install it in the cell, with the tools the operations you load require.
 email:
 - mike@37signals.com
 executables: []
 extensions: []
 extra_rdoc_files: []
-files: []
+files:
+- MIT-LICENSE
+- README.md
+- lib/active_storage/hot_cell/server.rb
+- lib/active_storage/hot_cell/server/analyzers/image/magick.rb
+- lib/active_storage/hot_cell/server/analyzers/image/vips.rb
+- lib/active_storage/hot_cell/server/analyzers/media/ffprobe.rb
+- lib/active_storage/hot_cell/server/magick_operation.rb
+- lib/active_storage/hot_cell/server/operation.rb
+- lib/active_storage/hot_cell/server/previewers/pdf.rb
+- lib/active_storage/hot_cell/server/previewers/pdf/mutool.rb
+- lib/active_storage/hot_cell/server/previewers/pdf/poppler.rb
+- lib/active_storage/hot_cell/server/previewers/video/ffmpeg.rb
+- lib/active_storage/hot_cell/server/tool_operation.rb
+- lib/active_storage/hot_cell/server/transformers/image/magick.rb
+- lib/active_storage/hot_cell/server/transformers/image/vips.rb
+- lib/active_storage/hot_cell/server/transforming.rb
+- lib/active_storage/hot_cell/server/version.rb
+- lib/active_storage/hot_cell/server/vips_operation.rb
+- lib/activestorage-hotcell-server.rb
+homepage: https://github.com/basecamp/hotcell
 licenses:
 - MIT
-metadata: {}
+metadata:
+  homepage_uri: https://github.com/basecamp/hotcell
+  source_code_uri: https://github.com/basecamp/hotcell/tree/v0.2.0/activestorage-hotcell-server
+  changelog_uri: https://github.com/basecamp/hotcell/blob/v0.2.0/CHANGELOG.md
+  bug_tracker_uri: https://github.com/basecamp/hotcell/issues
+  rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:
 - lib
@@ -26,7 +110,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '3.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -35,5 +119,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 rubygems_version: 4.0.16
 specification_version: 4
-summary: To be released soon, secure sidecar for Rails with Active Storage extensions
+summary: The operations a cell runs on behalf of Active Storage.
 test_files: []