monkrb 0.22.2 → 0.23.0

data/lib/monk/storage.rb

@@ -0,0 +1,184 @@
+require_relative "freeze_hooks"
+require_relative "environment"
+require_relative "log"
+require_relative "storage/errors"
+require_relative "storage/key"
+require_relative "storage/input"
+require_relative "storage/object"
+require_relative "storage/upload"
+require_relative "storage/token"
+require_relative "storage/routes"
+require_relative "storage/sig_v4"
+require_relative "storage/backends/file"
+require_relative "storage/backends/s3"
+require_relative "storage/url"
+require_relative "storage/sniff"
+
+module Monk
+  # Storing the files users send. Opt-in: require "monk/storage"
+  # explicitly -- `require "monk"` alone does not load this. One
+  # S3-compatible backend for every provider, local files in development
+  # and test, and uploads that go from the browser straight to the bucket
+  # (docs/adr/0018-storage-s3-compatible-direct-upload.md).
+  module Storage
+    # Signs the file backend's links in development and test when
+    # STORAGE_SECRET is unset. Fixed, so a link built by bin/jobs also
+    # works in the web process.
+    DEVELOPMENT_SECRET = "monk-storage-development-secret".freeze
+    # url's defaults: links valid for at least an hour, the same URL for
+    # a whole hour (docs/history/plan-storage.md decision 9).
+    CACHE_WINDOW = 3600
+    EXPIRES_IN = 3600
+    # presigned_put's default: an upload starts within minutes of asking.
+    UPLOAD_EXPIRES_IN = 600
+
+    class << self
+      # Typically, in config/storage.rb:
+      #
+      #   Monk::Storage.configure(url: ENV["STORAGE_URL"], secret: ENV["STORAGE_SECRET"])
+      #
+      # url: picks the backend (file:// or s3://) and is parsed here, so a
+      # bad one fails the boot. Unset or empty means file://storage/<env>
+      # in development and test, and raises in staging and production: on
+      # hosts whose disk is replaced on deploy, a silent fallback to local
+      # files would lose every upload. secret: signs the file backend's
+      # links, and is required with file:// outside development and test.
+      # An S3 backend doesn't use it. expires_in: and cache_window: are
+      # url's defaults, and together stay within the 7 days a signed S3
+      # link allows.
+      def configure(url:, secret: nil, expires_in: EXPIRES_IN, cache_window: CACHE_WINDOW)
+        Input.link_times!(expires_in: expires_in, cache_window: cache_window)
+        backend = backend_for(url)
+        backend = backend.with(secret: secret_for(secret)) if backend.is_a?(Backends::File)
+
+        @config = { backend: backend, expires_in: expires_in, cache_window: cache_window }
+      end
+
+      attr_reader :config
+
+      # "tmp/1f0c...": a random key under prefix. Uploads go under tmp/
+      # until the app confirms them.
+      def generate_key(prefix:)
+        Key.generate(prefix: prefix)
+      end
+
+      # The API, the same on every backend (docs/history/plan-storage.md decision 3),
+      # each call logged as one line without URLs or secrets.
+
+      # Returns the key. body is a String, or an IO that responds to size,
+      # read and rewind.
+      def put(key, body, content_type: Input::DEFAULT_CONTENT_TYPE)
+        logged("put", key) { [backend.put(key, body, content_type: content_type), "bytes=#{body.size}"] }
+      end
+
+      # The bytes, as a frozen binary String; range: 0..15 reads only the
+      # first 16. A missing key raises NotFoundError.
+      def get(key, range: nil)
+        logged("get", key) do
+          bytes = backend.get(key, range: range)
+          [bytes, "bytes=#{bytes.bytesize}"]
+        end
+      end
+
+      # A Storage::Object, or nil when the key is missing.
+      def head(key)
+        logged("head", key) do
+          object = backend.head(key)
+          [object, object ? "bytes=#{object.byte_size}" : "missing"]
+        end
+      end
+
+      # Idempotent: deleting a missing key is not an error.
+      def delete(key)
+        logged("delete", key) { [backend.delete(key), nil] }
+      end
+
+      # Returns to. A missing from raises NotFoundError.
+      def copy(from, to)
+        logged("copy", from) { [backend.copy(from, to), "to=#{to}"] }
+      end
+
+      # A signed download link for key, the String to redirect to (302)
+      # from the app's own route, after checking the viewer may see it.
+      # The same URL for every call within one cache_window, so the
+      # browser's cache works; valid at least expires_in seconds.
+      def url(key, expires_in: config_value(:expires_in), cache_window: config_value(:cache_window))
+        logged("url", key) { [backend.url(key, expires_in: expires_in, cache_window: cache_window), nil] }
+      end
+
+      # A signed upload link for a browser: an Upload with the url to PUT
+      # the file to and the headers to send. content_type: and byte_size:
+      # are inside the signature, so the bucket refuses any other. key
+      # goes under tmp/ until the app confirms it.
+      def presigned_put(key, content_type:, byte_size:, expires_in: UPLOAD_EXPIRES_IN)
+        logged("presigned_put", key) do
+          [backend.presigned_put(key, content_type: content_type, byte_size: byte_size, expires_in: expires_in),
+           "bytes=#{byte_size}",]
+        end
+      end
+
+      # Called from Base#dispatch, after Monk::Assets.response: the file
+      # backend's /_monk/storage routes, or nil for any other request
+      # (and always nil with an S3 backend, or before configure).
+      def response(env)
+        Routes.response(env, config[:backend]) if config
+      end
+
+      # Called from Base#freeze! (Seam B), via Monk.freeze_hooks. Freezes
+      # the value, not the module, same as Monk::Mail.freeze_registry! --
+      # an unfrozen config Hash can't be read from a worker Ractor at all.
+      def freeze_registry!
+        @config = Ractor.make_shareable(@config)
+      end
+
+      # Test-only: drops the current config. Not part of the app-facing API.
+      def reset!
+        @config = nil
+      end
+
+      private
+
+      def backend
+        raise NotConfiguredError, "call Monk::Storage.configure(url:) before using Monk::Storage" unless config
+
+        config[:backend]
+      end
+
+      def config_value(name)
+        backend
+        config[name]
+      end
+
+      # The block returns [result, detail for the log line].
+      def logged(operation, key)
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        result, detail = yield
+        elapsed = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round
+        Monk::Log.info("[storage] #{operation} key=#{key}#{" #{detail}" if detail} #{elapsed}ms")
+        result
+      end
+
+      def backend_for(url)
+        return URL.parse(url) unless url.nil? || url.empty?
+        return URL.parse("file://storage/#{Monk.env}") if Monk.env.development? || Monk.env.test?
+
+        raise MissingStorageUrlError,
+          "Monk::Storage.configure(url:) is empty and this isn't development or test (MONK_ENV=#{Monk.env}) -- " \
+          "uploads would go to local files a deploy can replace. Set STORAGE_URL, e.g. " \
+          "#{URL::S3_EXAMPLE}, or file:///path/on/a/volume if one server with a persistent volume is truly intended."
+      end
+
+      def secret_for(secret)
+        return secret unless secret.nil? || secret.empty?
+        return DEVELOPMENT_SECRET if Monk.env.development? || Monk.env.test?
+
+        raise MissingStorageSecretError,
+          "STORAGE_URL is file:// and Monk::Storage.configure(secret:) is empty (MONK_ENV=#{Monk.env}) -- " \
+          "the file backend's links would be signed with a public value. Set STORAGE_SECRET to a random " \
+          "value of its own (e.g. `ruby -rsecurerandom -e 'puts SecureRandom.hex(32)'`), not the auth secret."
+      end
+    end
+
+    Monk.freeze_hooks << self
+  end
+end

data/lib/monk/templates/base/Dockerfile

@@ -29,5 +29,10 @@ COPY . .
 # `bin/websocket_server` as the command instead of publishing another
 # port from this Dockerfile -- see docs/guides/deploying.md. Every other
 # process in bin/ (bin/jobs, say) is the same: this image, that command.
+# Production unless the platform says otherwise (MONK_ENV=staging in its
+# env, say). Unset, Monk runs as development: mail printed instead of
+# sent, magic links logged, demo routes on, a public storage secret.
+ENV MONK_ENV=production
+
 EXPOSE 9292
 CMD ["bin/server", "--bind", "0.0.0.0"]

data/lib/monk/templates/base/Gemfile

@@ -1,6 +1,6 @@
 source "https://rubygems.org"
 
-gem "monkrb", "~> 0.22", require: "monk"
+gem "monkrb", "~> 0.23", require: "monk"
 gem "rackup"
 gem "webrick"
 gem "kino"

data/lib/monk/templates/base/bin/server

@@ -2,4 +2,24 @@
 set -euo pipefail
 cd "$(dirname "$0")/.."
 
+# Bound beyond this machine (the Dockerfile's --bind 0.0.0.0) with MONK_ENV
+# unset: almost certainly deployed, and running as development. A warning,
+# not an error, so a container run locally on purpose still starts.
+bind="" previous=""
+for arg in "$@"; do
+  case "$previous" in -b | --bind) bind="$arg" ;; esac
+  case "$arg" in --bind=*) bind="${arg#--bind=}" ;; esac
+  previous="$arg"
+done
+case "$bind" in
+  "" | 127.* | localhost | ::1) ;;
+  *)
+    if [[ -z "${MONK_ENV:-}" ]] && ! grep -qs '^MONK_ENV=' .env; then
+      echo "bin/server: warning: listening on $bind with MONK_ENV unset, so this runs as development:" \
+        "mail is printed instead of sent, magic links are logged, auth cookies aren't Secure, demo routes" \
+        "are on. Set MONK_ENV=production (or staging) where the app is deployed." >&2
+    fi
+    ;;
+esac
+
 exec bundle exec kino -p "${PORT:-9292}" "$@" config.ru

data/lib/monk/templates/base/setup.md

@@ -7,6 +7,16 @@ bundle install
 bin/server              # the app on http://localhost:9292
 ```
 
+## Before production
+
+Set `MONK_ENV=production` (or `staging`) wherever the app runs deployed.
+Unset, it means `development`, and the app runs that way without an
+error: mail printed instead of sent, magic links logged, demo routes on.
+The `Dockerfile` already sets it; without the Dockerfile, set it in the
+platform's env. `bin/server` warns when it listens beyond this machine
+(`--bind 0.0.0.0`) with `MONK_ENV` unset. Platform examples:
+`docs/guides/deploying.md` in the monk repo.
+
 ## Check it works
 
 1. `bundle exec rake test` runs the tests.

data/lib/monk/templates/storage/agents.md

@@ -0,0 +1,35 @@
+## storage
+
+**Where:** `config/storage.rb` (from `STORAGE_URL`, `STORAGE_SECRET`);
+routes that upload and serve files in `app/routes/storage.rb`. Files live
+under `storage/<env>` until `STORAGE_URL` names a bucket.
+
+**Main calls:** `Monk::Storage.generate_key(prefix: "tmp")` for every new
+file; `put(key, body, content_type:)`, `get(key, range: 0..99)`,
+`head(key)` (an object with `byte_size`, `content_type`, `etag`, or nil),
+`delete(key)`, `copy(from, to)`. Downloads: `redirect
+Monk::Storage.url(key)`. Browser uploads:
+`presigned_put(key, content_type:, byte_size:)` gives the `url` and
+`headers` to PUT the file with. At confirm,
+`Monk::Storage.get(key, range: Monk::Storage::Sniff::RANGE)` and
+`Monk::Storage::Sniff.content_type(bytes)` check the file is what it
+claims; `Sniff.refuse_svg!(bytes)` raises on SVG.
+
+**Test:** `test/storage_test.rb`. Tests use `storage/test` (an unset
+`STORAGE_URL`); call a signed link through `APP.call` with
+`Rack::MockRequest.env_for(Monk::Storage.url(key))`.
+
+**Pitfalls:**
+- Keys come from `generate_key`, never from a user's filename: keep the
+  filename in the app's own table.
+- An upload stays under `tmp/` until the app checks it; then `copy` to
+  its final key, and `delete` the `tmp/` one after the copy, never before.
+- Check the viewer may see a file before redirecting to its `url`: a
+  signed link works for anyone who has it, until it expires.
+- Large files go from the browser straight to the bucket with
+  `presigned_put`. Don't stream them through a route: it holds a worker
+  Ractor for the whole upload.
+- Never accept SVG or HTML as an upload: both can carry scripts.
+
+**Examples:** `storage-upload`, `storage-confirm` and `storage-download`
+in `app/routes/storage.rb`.

data/lib/monk/templates/storage/app/routes/storage.rb

@@ -0,0 +1,67 @@
+# Routes that store files users send (monk add storage). An upload goes
+# from the browser straight to the bucket (or, in development, to
+# /_monk/storage): bin/server only signs the link, then checks the file.
+#
+# monk:example storage-upload -- a signed link to upload one file to
+# # Only the types and sizes the app accepts get a link, and both are
+# # inside its signature. The file goes under tmp/ until it's confirmed.
+# # Try it (in development the link is a path on this app):
+# #   curl -X POST http://localhost:9292/uploads -H "content-type: application/json" \
+# #     -d "{\"content_type\": \"image/png\", \"byte_size\": $(wc -c < photo.png)}"
+# #   curl -X PUT "http://localhost:9292<url>" -H "content-type: image/png" --data-binary @photo.png
+# class App
+#   post("/uploads") do
+#     content_type = params[:content_type].to_s
+#     byte_size = params[:byte_size].to_i
+#     halt 422, "type not accepted" unless %w[image/png image/jpeg image/webp application/pdf].include?(content_type)
+#     halt 422, "size not accepted" unless byte_size.between?(1, 10 * 1024 * 1024)
+#
+#     key = Monk::Storage.generate_key(prefix: "tmp")
+#     upload = Monk::Storage.presigned_put(key, content_type: content_type, byte_size: byte_size)
+#     json(key: key, url: upload.url, headers: upload.headers)
+#   end
+# end
+# monk:end
+#
+# monk:example storage-confirm -- check an upload, then move it out of tmp/
+# # head, check the first bytes, copy, then delete: if a step fails, the
+# # tmp/ file stays and the bucket's lifecycle rule expires it. A real app
+# # also checks who uploaded it, and saves a record (key, filename, type,
+# # size) in its own table: docs/guides/storage.md in the monk repo.
+# #   curl -X POST http://localhost:9292/uploads/confirm -H "content-type: application/json" \
+# #     -d '{"key": "tmp/<key>"}'
+# class App
+#   post("/uploads/confirm") do
+#     key = params[:key].to_s
+#     halt 422, "not an upload" unless key.match?(%r{\Atmp/[0-9a-f]{32}\z})
+#     object = Monk::Storage.head(key)
+#     halt 404, "no such upload" unless object
+#
+#     # Also refuses SVG, which can carry scripts: Sniff never names it.
+#     first_bytes = Monk::Storage.get(key, range: Monk::Storage::Sniff::RANGE)
+#     unless Monk::Storage::Sniff.content_type(first_bytes) == object.content_type
+#       Monk::Storage.delete(key)
+#       halt 422, "the file isn't a #{object.content_type}"
+#     end
+#
+#     media_key = key.sub("tmp/", "media/")
+#     Monk::Storage.copy(key, media_key)
+#     Monk::Storage.delete(key)
+#     json(key: media_key, content_type: object.content_type, byte_size: object.byte_size)
+#   end
+# end
+# monk:end
+#
+# monk:example storage-download -- a redirect to a file's signed link
+# # NO ACCESS CHECK: anyone who knows a key gets the file. In a real app
+# # this becomes GET /media/:id, which looks the record up, checks the
+# # viewer may see it, then redirects to its key's link.
+# #   curl -i http://localhost:9292/media/<key without media/>
+# class App
+#   get("/media/*") do
+#     halt 404, "" unless params[:splat].match?(/\A[0-9a-f]{32}\z/)
+#
+#     redirect Monk::Storage.url("media/#{params[:splat]}")
+#   end
+# end
+# monk:end

data/lib/monk/templates/storage/config/storage.rb

@@ -0,0 +1,21 @@
+require "monk"
+require "monk/storage"
+
+# Stores the files users send with Monk::Storage: put, get, head, delete
+# and copy by key, signed download links (url) and uploads that go from
+# the browser straight to the bucket (presigned_put). See
+# docs/guides/storage.md in the monk repo. Loaded by config/load.rb.
+Monk::Settings.configure do
+  # Where files go. Unset in development and test means
+  # file://storage/<env>, files under this app's storage/ directory, which
+  # Monk serves at /_monk/storage; anywhere else an unset one fails the
+  # boot. In production an S3 bucket, e.g.
+  # s3://KEY:SECRET@bucket?endpoint=fsn1.your-objectstorage.com&region=fsn1
+  optional :storage_url, default: ""
+  # Signs the file backend's links. Only needed with a file:// URL outside
+  # development and test (one server with a persistent volume); a bucket
+  # signs its own. A random value of its own, not AUTH_SECRET.
+  optional :storage_secret, default: ""
+end
+
+Monk::Storage.configure(url: Monk::Settings[:storage_url], secret: Monk::Settings[:storage_secret])

data/lib/monk/templates/storage/setup.md

@@ -0,0 +1,43 @@
+## storage
+
+Files users send, stored by key. In development and test `STORAGE_URL` is
+unset, so files go under this app's `storage/development` and
+`storage/test` directories (ignored by git), and Monk serves their signed
+links at `/_monk/storage`, the way a bucket would. Nothing to start.
+
+### Before production
+
+Create an S3-compatible bucket (Hetzner Object Storage, AWS S3, Cloudflare
+R2, ...) and set `STORAGE_URL`, e.g.
+`s3://KEY:SECRET@bucket?endpoint=fsn1.your-objectstorage.com&region=fsn1`.
+An unset `STORAGE_URL` fails the boot outside development and test. The
+bucket needs a CORS rule allowing `PUT` from `PUBLIC_URL`, so browsers can
+upload to it, and a lifecycle rule expiring `tmp/` after a day, so
+unconfirmed uploads don't pile up: both are in docs/guides/storage.md in
+the monk repo. A `file://` URL works only on one server with a persistent
+volume, and then needs `STORAGE_SECRET` set to a random value of its own.
+
+### Check it works
+
+1. `bundle exec rake test` runs `test/storage_test.rb`: a file stored by
+   key reads back, and its signed link answers.
+2. By hand:
+   `bundle exec ruby -r ./config/load -e 'Monk::Storage.put("notes/hello", "Hello"); p Monk::Storage.get("notes/hello")'`
+   prints `"Hello"`, and the file is under `storage/development/objects/`.
+3. The examples: uncomment `storage-upload`, `storage-confirm` and
+   `storage-download` in `app/routes/storage.rb`, restart `bin/server`,
+   and upload a PNG:
+
+   ```bash
+   curl -X POST http://localhost:9292/uploads -H "content-type: application/json" \
+     -d "{\"content_type\": \"image/png\", \"byte_size\": $(wc -c < photo.png)}"
+   # PUT the file to the "url" it answers, with the content-type it signed:
+   curl -X PUT "http://localhost:9292<url>" -H "content-type: image/png" --data-binary @photo.png
+   curl -X POST http://localhost:9292/uploads/confirm -H "content-type: application/json" \
+     -d '{"key": "<key>"}'
+   # The confirmed key is media/<hex>; open its link:
+   curl -iL http://localhost:9292/media/<hex>
+   ```
+
+   The last command follows a redirect to a signed `/_monk/storage` link
+   and gets the PNG back.

data/lib/monk/templates/storage/test/storage_test.rb

@@ -0,0 +1,33 @@
+require_relative "test_helper"
+require "rack/mock"
+
+# Monk::Storage is configured, and with STORAGE_URL unset tests use
+# file://storage/test, apart from development's files: a file stored by
+# key reads back, and its signed link answers. SETUP.md, storage.
+class StorageTest < Minitest::Test
+  def setup
+    @key = Monk::Storage.generate_key(prefix: "check")
+  end
+
+  def teardown
+    Monk::Storage.delete(@key)
+  end
+
+  def test_a_file_stored_by_key_reads_back
+    Monk::Storage.put(@key, "hello", content_type: "text/plain")
+
+    object = Monk::Storage.head(@key)
+    assert_equal ["text/plain", 5], [object.content_type, object.byte_size]
+    assert_equal "hello", Monk::Storage.get(@key)
+  end
+
+  def test_its_signed_link_answers
+    Monk::Storage.put(@key, "hello", content_type: "text/plain")
+
+    status, headers, body = APP.call(Rack::MockRequest.env_for(Monk::Storage.url(@key)))
+
+    assert_equal 200, status
+    assert_equal "text/plain", headers["content-type"]
+    assert_equal "hello", body.to_enum(:each).to_a.join
+  end
+end

data/lib/monk/version.rb

@@ -5,5 +5,5 @@ module Monk
   # class of bug .freeze! guards against for routes/error handlers/models,
   # just on a top-level constant that every worker Ractor reads on boot
   # (found live under kino: GET /hello 500'd until this was frozen).
-  VERSION = "0.22.2".freeze
+  VERSION = "0.23.0".freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: monkrb
 version: !ruby/object:Gem::Version
-  version: 0.22.2
+  version: 0.23.0
 platform: ruby
 authors:
 - Matteo Folin
@@ -206,6 +206,7 @@ files:
 - lib/monk/generators/mail.rb
 - lib/monk/generators/postgres.rb
 - lib/monk/generators/redis.rb
+- lib/monk/generators/storage.rb
 - lib/monk/generators/websocket.rb
 - lib/monk/jobs.rb
 - lib/monk/jobs/adapters/pg.rb
@@ -251,6 +252,19 @@ files:
 - lib/monk/persistence/pool.rb
 - lib/monk/settings.rb
 - lib/monk/state_ractor.rb
+- lib/monk/storage.rb
+- lib/monk/storage/backends/file.rb
+- lib/monk/storage/backends/s3.rb
+- lib/monk/storage/errors.rb
+- lib/monk/storage/input.rb
+- lib/monk/storage/key.rb
+- lib/monk/storage/object.rb
+- lib/monk/storage/routes.rb
+- lib/monk/storage/sig_v4.rb
+- lib/monk/storage/sniff.rb
+- lib/monk/storage/token.rb
+- lib/monk/storage/upload.rb
+- lib/monk/storage/url.rb
 - lib/monk/templates/auth/agents.md
 - lib/monk/templates/auth/app/mailers/auth_mailer.rb
 - lib/monk/templates/auth/app/routes/auth.rb
@@ -334,6 +348,11 @@ files:
 - lib/monk/templates/redis/config/redis.rb
 - lib/monk/templates/redis/setup.md
 - lib/monk/templates/redis/test/redis_test.rb
+- lib/monk/templates/storage/agents.md
+- lib/monk/templates/storage/app/routes/storage.rb
+- lib/monk/templates/storage/config/storage.rb
+- lib/monk/templates/storage/setup.md
+- lib/monk/templates/storage/test/storage_test.rb
 - lib/monk/templates/websocket/agents.md
 - lib/monk/templates/websocket/app/sockets/chat.rb
 - lib/monk/templates/websocket/bin/websocket_server