findxpand 0.1.0

data/lib/findxpand/cli.rb

@@ -0,0 +1,214 @@
+# frozen_string_literal: true
+
+# `findxpand init | doctor | status` — three commands, and no network.
+#
+# **Nothing in this file makes an HTTP request, including to localhost.** The
+# gem's one architectural promise is that it never calls out: Findxpand pushes to
+# the customer's origin and the customer's origin never fetches from Findxpand,
+# so that our uptime is never their uptime. A CLI that quietly curled something
+# would be the first exception, and the exception is how the promise stops being
+# true. `status` therefore reads the cache file this machine holds and prints the
+# `curl` line for the endpoint rather than calling it — which is also the only
+# honest thing it can do, since this process does not know which port the server
+# is on.
+#
+# ## What `doctor` can and cannot tell you
+#
+# It runs in a *different process* from the server, so it cannot answer the only
+# question that finally matters — "is the middleware in the request path" — and
+# it says so rather than implying otherwise. Everything it checks is a
+# precondition. `/__findxpand/status` is the thing that knows, and `doctor`
+# prints the command for asking it. §20.1 rule 3: a tool whose job is to catch
+# failure must not report health for something it never examined, and the honest
+# form of that here is naming what was out of reach.
+#
+# Frozen string literals: on. Output goes through `puts`, never through `<<` on a
+# literal.
+
+require 'findxpand'
+require 'json'
+
+module Findxpand
+  module CLI
+    USAGE = <<~TEXT
+      usage: findxpand <command>
+
+        init     print the lines to add to your app, ready to paste
+        doctor   check this machine's preconditions, and say what it cannot check
+        status   print the manifest this machine has cached
+
+      FINDXPAND_TOKEN must be set in the same environment as your server.
+    TEXT
+
+    def self.run(argv, out: $stdout, err: $stderr, env: ENV)
+      case argv.first
+      when 'init'
+        init(out, env)
+      when 'doctor'
+        doctor(out, env)
+      when 'status'
+        status(out, err, env)
+      when nil, '-h', '--help', 'help'
+        out.puts(USAGE)
+        0
+      when '-v', '--version'
+        out.puts(VERSION)
+        0
+      else
+        err.puts("findxpand: unknown command #{argv.first.inspect}")
+        err.puts(USAGE)
+        2
+      end
+    end
+
+    def self.init(out, env)
+      token = env['FINDXPAND_TOKEN'].to_s.empty? ? '<we send you this>' : '<already set>'
+      out.puts('# 1. the gem.')
+      out.puts("#    #{UNPUBLISHED_NOTE}") unless PUBLISHED_TO_REGISTRIES
+      # `Findxpand.` spelled out. Constants resolve up the lexical scope and
+      # methods do not, so a bare `install_command` here is a NameError rather
+      # than the enclosing module's method.
+      out.puts("#    #{Findxpand.install_command}")
+      out.puts
+      out.puts('# in your Gemfile, so that `bundle exec` can find it. This line is not')
+      out.puts('# optional: without it the RUBYOPT below raises LoadError and your server')
+      out.puts('# does not start at all.')
+      out.puts("gem 'findxpand'")
+      out.puts
+      out.puts('# 2. the attach. One prefix on the command you already run:')
+      out.puts("#{ATTACH} bundle exec puma")
+      out.puts
+      out.puts('# 3. the environment your server runs in:')
+      out.puts("FINDXPAND_TOKEN=#{token}")
+      out.puts('FINDXPAND_CACHE_FILE=/var/lib/findxpand/manifest.json')
+      out.puts
+      out.puts('# The cache path must survive a restart, and must be shared by every worker')
+      out.puts('# if your server forks (Puma in cluster mode, Unicorn, Passenger). Without')
+      out.puts('# it a restart drops every rule until the next push, and a push reaches one')
+      out.puts('# worker out of however many you run.')
+      out.puts
+      # Printed commented-out, and printed at all because the README listed the
+      # other five variables and omitted this one until 2 Sep 2026. It controls
+      # the whole Accept-Encoding takeover this gem is built around, and a
+      # customer behind an intermediary that has already taken over content
+      # negotiation - the one supported reason to turn it off, named in
+      # `middleware.rb` - had no way to learn the variable existed. What they
+      # reach for instead is FINDXPAND_ENABLED, which turns off every rewrite on
+      # the site.
+      out.puts('# Only if something in front of your app has already taken over content')
+      out.puts('# negotiation. Off means a page with a rule is served uncompressed; it')
+      out.puts('# does not change what is rewritten.')
+      out.puts('# FINDXPAND_RECOMPRESS=off')
+      0
+    end
+
+    def self.doctor(out, env)
+      results = []
+      results << check('ruby >= 3.0', RUBY_VERSION.split('.').first.to_i >= 3,
+                       "this is Ruby #{RUBY_VERSION}")
+      results << check('FINDXPAND_TOKEN is set', !env['FINDXPAND_TOKEN'].to_s.empty?,
+                       'without it nothing attaches and the admin endpoints 401 everybody')
+      # Worded as "a Rack builder" rather than "Rack::Builder", because the attach
+      # stopped being one class on 2 Sep 2026: it prepends every loaded builder by
+      # name and watches for the one a server defines for itself (Puma vendors
+      # one). Rack being loadable is still the honest precondition to check here -
+      # it is the one this process can see - and it is no longer the whole
+      # mechanism, so it must not be described as if it were.
+      results << check('rack is loadable', rack_loadable?,
+                       'the zero-code attach prepends a Rack builder; without Rack in ' \
+                       'this process there is none to find here')
+      results << check('RUBYOPT carries findxpand/auto',
+                       env['RUBYOPT'].to_s.include?('findxpand/auto'),
+                       'checked for THIS process only - the one that matters is your server\'s')
+      results << cache_check(env)
+
+      out.puts('findxpand doctor')
+      out.puts
+      results.each { |line, _ok| out.puts("  #{line}") }
+      out.puts
+      out.puts('This process is not your server, so nothing above can tell you whether the')
+      out.puts('middleware is in the request path. One thing can:')
+      out.puts
+      out.puts('    curl -H "Authorization: Bearer $FINDXPAND_TOKEN" \\')
+      out.puts("         http://localhost:3000#{Admin::STATUS_PATH}")
+      out.puts
+      out.puts('Read `degraded`, `degraded_reason`, `persisted` and `attach`. A process')
+      out.puts('that has been up a while with `requests_seen: 0` is attached somewhere')
+      out.puts('requests do not go, and reports itself degraded rather than healthy;')
+      out.puts('`persisted: false` means a push was taken and could not be written, so a')
+      out.puts('restart drops it; `attach.built: 0` means this process placed its hook and')
+      out.puts('the hook never ran - the server built its application some other way.')
+      results.all? { |_line, ok| ok } ? 0 : 1
+    end
+
+    def self.status(out, err, env)
+      path = env['FINDXPAND_CACHE_FILE'].to_s
+      if path.empty?
+        err.puts('findxpand: FINDXPAND_CACHE_FILE is not set, so there is no manifest on ' \
+                 'this machine to read.')
+        return 1
+      end
+      unless File.exist?(path)
+        # Not an error in the sense of a bug: a fresh install has been pushed
+        # nothing yet. Said in words rather than by printing an empty table,
+        # which would read as "no rules" when it means "no file".
+        err.puts("findxpand: no manifest cached at #{path} yet - nothing has been pushed " \
+                 'to this machine, or the server writes it somewhere else.')
+        return 1
+      end
+
+      begin
+        manifest = JSON.parse(File.read(path, encoding: 'UTF-8'))
+      rescue StandardError => e
+        err.puts("findxpand: #{path} is not readable as a manifest (#{e.class}).")
+        return 1
+      end
+      unless Store.manifest?(manifest)
+        err.puts("findxpand: #{path} parsed but is not a manifest - expected {version, pages}.")
+        return 1
+      end
+
+      pages = manifest['pages']
+      redirects = manifest['redirects'].is_a?(Array) ? manifest['redirects'].size : 0
+      out.puts("version   #{manifest['version']}")
+      out.puts("pages     #{pages.size}")
+      out.puts("redirects #{redirects}")
+      out.puts("written   #{File.mtime(path)}")
+      out.puts
+      pages.keys.sort.each do |key|
+        fields = pages[key].is_a?(Hash) ? pages[key].keys.sort.join(', ') : ''
+        out.puts("  #{key}  #{fields}")
+      end
+      out.puts
+      out.puts('This is the file on disk. Whether the running process is serving it is a')
+      out.puts("different question, and only #{Admin::STATUS_PATH} answers it.")
+      0
+    end
+
+    def self.check(name, ok, detail)
+      line = ok ? "ok    #{name}" : "FAIL  #{name} - #{detail}"
+      [line, ok]
+    end
+
+    def self.cache_check(env)
+      path = env['FINDXPAND_CACHE_FILE'].to_s
+      if path.empty?
+        return check('FINDXPAND_CACHE_FILE is set', false,
+                     'without it a restart drops every rule until the next push, and on a ' \
+                     'forking server a push reaches one worker')
+      end
+
+      directory = File.dirname(path)
+      writable = File.directory?(directory) && File.writable?(directory)
+      check("#{directory} is writable", writable,
+            'the manifest cannot be persisted there, so a restart drops every rule')
+    end
+
+    def self.rack_loadable?
+      require 'rack'
+      defined?(::Rack::Builder) ? true : false
+    rescue LoadError
+      false
+    end
+  end
+end

data/lib/findxpand/encoding.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+# Taking the response uncompressed, and giving it back compressed.
+#
+# ## The bug this exists to close
+#
+# The install screen told customers to mount the middleware outermost. In Rack
+# that is `insert 0` or the first `use` in `config.ru`, and it is the only place
+# a wrapper of this kind can go — and on its own it is **wrong**. `use
+# Rack::Deflater` inside our position means the body we are handed has already
+# been gzipped, so the rewrite declines, marks the response `skip-encoded` and
+# serves the customer's original title. The same measurement was taken against
+# Django's `GZipMiddleware` and Express's `compression` on 31 Aug 2026:
+#
+#     marker         skip-encoded
+#     title served   <title>Original</title>
+#     /status        healthy
+#
+# Nothing failed. That is the whole problem: the customer deployed, the screen
+# said connected, `/status` said fine, and the fix never landed. There is no
+# mount position that fixes it either — inside the compression we are no longer
+# outermost and miss half the responses, and in Express mounting the other way
+# round put 2069 bytes of plain HTML on the wire under `content-encoding: gzip`.
+# The instruction had no right answer.
+#
+# ## What replaces it
+#
+# The wrapper stops asking to be placed correctly and takes control of the
+# negotiation instead. For a path the manifest names, and only for those:
+#
+#     inbound    env['HTTP_ACCEPT_ENCODING'] is replaced with `identity`, so the
+#                application and every middleware it wraps produce real HTML
+#     outbound   the rewritten page is compressed here, with the coding the
+#                client actually asked for
+#
+# `Rack::Deflater` reads `env['HTTP_ACCEPT_ENCODING']` *after* calling the app,
+# so overwriting it on the way down is enough to make it stand aside; and a
+# `Rack::Deflater` mounted *outside* us stands aside too, because it declines a
+# response that already carries a `content-encoding`. This is the sidecar's
+# technique — `proxy.ts` asks the origin for `identity` for the identical reason
+# — applied inside the process.
+#
+# A path with no rule keeps its own `Accept-Encoding` untouched and is compressed
+# by the application exactly as before, which is what keeps the cost of turning
+# this on at zero.
+#
+# ## Only gzip
+#
+# The standard library has zlib and no brotli, and this gem has no dependencies
+# on purpose — one that drags in a build-time C extension is one that can break
+# somebody's deploy. Every client that offers `br` offers `gzip` alongside it, so
+# the practical cost is a few percent on the wire for the handful of pages we
+# rewrite. A client that offers brotli *alone* gets identity, which is correct
+# and merely larger. This mirrors the Python package exactly; the Node one adds
+# brotli because Node ships a brotli in its standard library.
+#
+# Frozen string literals: on. The compressed body is built in a `StringIO`, never
+# by appending to a literal.
+
+require 'zlib'
+require 'stringio'
+
+module Findxpand
+  module Encoding
+    # Below this a compressed body is usually the larger one. Matches the Node
+    # and Python sides' `MIN_COMPRESS_BYTES` so the three packages do not
+    # disagree about whether a given page came back compressed.
+    MIN_COMPRESS_BYTES = 1024
+
+    # What we can actually produce. An Array so the negotiation below reads as a
+    # preference order rather than a special case.
+    SUPPORTED = %w[gzip].freeze
+
+    # The best coding we can produce from what the client offered, or `''`.
+    #
+    # `;q=0` is read, because it is a refusal rather than an offer and sending a
+    # coding a client has explicitly declined is a blank page. Anything else in
+    # the q-value grammar is ignored: the client sent a list of what it accepts
+    # and picking a supported entry out of it is the entire job here.
+    def self.negotiate(accept)
+      offered = {}
+      accept.to_s.downcase.split(',').each do |part|
+        token, _, params = part.partition(';')
+        name = token.strip
+        next if name.empty?
+
+        refused = params.split(';').any? { |piece| piece.strip.start_with?('q=') && q(piece).zero? }
+        offered[name] = true unless refused
+      end
+      SUPPORTED.find { |coding| offered[coding] } || ''
+    end
+
+    # A q-value, or 1.0 when it is not a number.
+    #
+    # `Float()` and not `to_f` for `signature.rb`'s reason: `'q=nonsense'.to_f`
+    # is `0.0`, which reads as a *refusal* — so a malformed parameter would
+    # silently stop us compressing for that client rather than being ignored.
+    def self.q(piece)
+      Float(piece.strip[2..].to_s)
+    rescue ArgumentError, TypeError
+      1.0
+    end
+
+    # Compress, or `nil` when it should not or could not be done.
+    #
+    # `nil` rather than an exception, and `nil` for a body under the threshold as
+    # well as for a compressor that raised: the caller's answer to both is the
+    # same one, and it is the safe one — serve the bytes we were given.
+    def self.encode(body, coding)
+      body = body.to_s
+      return nil unless SUPPORTED.include?(coding)
+      # Bytes, not characters. A 900-character Arabic page is about 1.6 KB on
+      # the wire, and `length` would decline to compress it while `bytesize`
+      # measures what actually travels (§14).
+      return nil if body.bytesize < MIN_COMPRESS_BYTES
+
+      buffer = StringIO.new(String.new('', encoding: ::Encoding::BINARY))
+      writer = Zlib::GzipWriter.new(buffer)
+      # Set before the first write, which is the only time zlib accepts it. The
+      # same page must compress to the same bytes, or every response is a cache
+      # miss and an ETag nobody can match — the reason the Python side passes
+      # `mtime=0` to `gzip.compress`.
+      writer.mtime = 0
+      writer.write(body)
+      # `finish`, not `close`: it writes the gzip trailer and hands back the
+      # underlying IO without closing it, so `buffer.string` is unambiguously
+      # readable afterwards. `close` would close the StringIO as well, and
+      # whether that is safe is a detail of StringIO nobody should have to know.
+      writer.finish
+      buffer.string
+    rescue StandardError, Zlib::Error
+      # The application's bytes are still a correct answer. Every failure in this
+      # gem goes in the same direction: unmodified, never wrong.
+      nil
+    end
+
+    # Did the application tell intermediaries not to re-encode this?
+    #
+    # Honoured, because the one thing worse than an uncompressed page is a
+    # compressed one that the application explicitly asked nobody to make.
+    def self.no_transform?(cache_control)
+      /no-transform/i.match?(cache_control.to_s)
+    end
+
+    # The `Vary` this response needs, or `''` when it already says it.
+    #
+    # The rule rather than the edit, because a Rack 2 header hash and a Rack 3
+    # one differ in case and a WSGI header list differs from both, and two copies
+    # of one rule is the shape §20.1 rule 2 is about. Every caller asks here and
+    # splices into its own structure.
+    #
+    # A shared cache that missed this serves gzip to the next client through it,
+    # including one that told us it cannot read gzip.
+    def self.vary_value(existing)
+      return '' if /accept-encoding/i.match?(existing.to_s)
+
+      existing.to_s.empty? ? 'Accept-Encoding' : "#{existing}, Accept-Encoding"
+    end
+  end
+end