gemstack 0.2.5 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 47837a1dc3df55eebfe8a54a2fdec217c6fa7821ed78fcad80f786bf0e2e2070
-  data.tar.gz: f3e13a9280d9aec92a8aa9739cdd55f360a2c46caacc4b5c3c188c1fb77bc17a
+  metadata.gz: 72fb1f0292d7127940eaf3eab4c5e701a4150c43278fb2579065685700b60ff1
+  data.tar.gz: be15ee4fb994153c86320b4218adcfa31bd5e94db1d4a14f9e9e38ecd528338a
 SHA512:
-  metadata.gz: 7efc4225b421fda7a3e9c1bc133a80a131d2d7cfcc34b1549bd4d9778b16ba247dc7689bca4315fb3c27ecf5ce86ebbe23cb09db88fbb361b21589c12296aad9
-  data.tar.gz: dbc2d8f2f0a1ced22395d1c095802846f61ef04c85657c7a8ea12875d166907fb3320cfd0fe4f9ebfd814e93991538670895a53997a965168fbb5f40a3ed81c1
+  metadata.gz: ed1794c6bc7e19ef464e152b166d901a06f9ee05f0c3e5fa3b2f56e2531a92208250b321704f357b2d04e53f482acba05208edd0b4ff9cd4a3d85a1427b912bc
+  data.tar.gz: 5b25b7a0c6e7758ee015f541907df8556fe8160c15558db7dbaa0c1fa1aad451cfcf324d5f3e8e80960193c8166eb39296005cd073af240e3812eaaad5725ea3

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.3.0
+
+The framework gem now contains core, cache, schema, http, db, jobs, mail, storage, contract, dev and the CLI code. See the [GemStack changelog](https://github.com/gemstack-rb/gemstack/blob/main/CHANGELOG.md).
+
 ## 0.2.5
 
 See the [GemStack changelog](https://github.com/gemstack-rb/gemstack/blob/main/CHANGELOG.md).

data/README.md

@@ -3,8 +3,8 @@
 GemStack: a fast, modular Ruby API framework for Next.js applications.
 
 Part of [GemStack](https://github.com/gemstack-rb/gemstack), a modular Ruby API framework for Next.js
-applications by [Adware Technologies](https://www.adwaretech.com). All GemStack gems are developed together in that repository and released with the same
-version.
+applications by [Adware Technologies](https://www.adwaretech.com). All GemStack gems are developed
+together in that repository and released with the same version.
 
 ## Installation
 
@@ -13,7 +13,11 @@ gem install gemstack
 gemstack new myapp
 ```
 
-This gem is the framework: it depends on every module an app needs (core, cache, schema, http, contract, dev, cli). Optional modules are added per app.
+This gem is the framework — routing, controllers, models (SQLite, PostgreSQL, MySQL), background jobs,
+mail, file storage, the TypeScript contract, generators and the development server. Apps switch modules
+on in `config/app.rb` (`require "gemstack/db"`, …). Authentication and realtime are the
+[gemstack-auth](https://rubygems.org/gems/gemstack-auth) and
+[gemstack-realtime](https://rubygems.org/gems/gemstack-realtime) gems.
 
 ## Documentation
 

data/lib/gemstack/cache/memory_store.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module GemStack
+  module Cache
+    # A thread-safe, per-process LRU cache with expiry. Values are stored
+    # marshalled, so callers never share (and can't mutate) cached objects —
+    # the same semantics as the Redis store. With several Puma workers each
+    # has its own copy; use Redis to share a cache across processes.
+    class MemoryStore < Store
+      Entry = Struct.new(:data, :expires_at)
+
+      def initialize(max_entries: 10_000, **)
+        super(**)
+        @max_entries = max_entries
+        @entries = {}
+        @mutex = Mutex.new
+      end
+
+      def size = @entries.size
+      def clear = @mutex.synchronize { @entries.clear }
+
+      def increment(key, by = 1, expires_in: default_expires_in)
+        normalized = normalize(key)
+        @mutex.synchronize do
+          entry = live_entry(normalized)
+          value = (entry ? Integer(load(entry.data)) : 0) + by
+          store(normalized, value, entry ? entry.expires_at : expires_at(expires_in))
+          value
+        end
+      end
+
+      private
+
+      def read_entry(key)
+        @mutex.synchronize do
+          entry = live_entry(key) or return MISSING
+          @entries[key] = @entries.delete(key) # mark as most recently used
+          load(entry.data)
+        end
+      end
+
+      def write_entry(key, value, expires_in)
+        data = dump(value)
+        @mutex.synchronize { store_data(key, data, expires_at(expires_in)) }
+      end
+
+      def delete_entry(key) = @mutex.synchronize { !@entries.delete(key).nil? }
+
+      # Callers hold the mutex.
+      def live_entry(key)
+        entry = @entries[key]
+        return nil unless entry
+        return entry unless entry.expires_at && entry.expires_at <= monotonic
+
+        @entries.delete(key)
+        nil
+      end
+
+      def store(key, value, expires_at) = store_data(key, dump(value), expires_at)
+
+      def store_data(key, data, expires_at)
+        @entries.delete(key)
+        @entries[key] = Entry.new(data, expires_at)
+        @entries.shift while @entries.size > @max_entries # Hash order = LRU order
+        true
+      end
+    end
+  end
+end

data/lib/gemstack/cache/null_store.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module GemStack
+  module Cache
+    # Caches nothing: fetch always runs its block. The default in tests, so
+    # tests never depend on state cached by earlier tests.
+    class NullStore < Store
+      def clear = true
+      def increment(_key, by = 1, **) = by
+
+      private
+
+      def read_entry(_key) = MISSING
+      def write_entry(_key, _value, _expires_in) = true
+      def delete_entry(_key) = false
+    end
+  end
+end

data/lib/gemstack/cache/redis_store.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module GemStack
+  module Cache
+    # A cache shared by every process, through Redis. Uses the `redis-client`
+    # gem (add `gem "redis-client"` to the Gemfile) with a connection pool
+    # sized to the server's threads, or any client object you pass that
+    # responds to #call(*command) (a redis-client pool or connection).
+    #
+    #   config.cache.store = :redis                      # REDIS_URL
+    #   config.cache.store = GemStack::Cache::RedisStore.new(client: my_pool, namespace: "shop")
+    class RedisStore < Store
+      attr_reader :client
+
+      def initialize(url: nil, client: nil, pool_size: 5, **)
+        super(**)
+        @client = client || build_client(url, pool_size)
+      end
+
+      def increment(key, by = 1, expires_in: default_expires_in)
+        normalized = normalize(key)
+        value = client.call("INCRBY", normalized, by)
+        client.call("PEXPIRE", normalized, (Float(expires_in) * 1000).round, "NX") if expires_in
+        value
+      end
+
+      # Deletes only keys in this store's namespace (never FLUSHDB).
+      def clear
+        raise Error, "RedisStore#clear needs a namespace (config.cache.namespace)" unless namespace
+
+        cursor = "0"
+        loop do
+          cursor, keys = client.call("SCAN", cursor, "MATCH", "#{namespace}:*", "COUNT", 500)
+          client.call("UNLINK", *keys) unless keys.empty?
+          break if cursor == "0"
+        end
+        true
+      end
+
+      private
+
+      def read_entry(key)
+        data = client.call("GET", key)
+        return MISSING if data.nil?
+
+        # Counters are stored as plain integers so INCRBY works on them.
+        data.match?(/\A-?\d+\z/) ? Integer(data) : load(data.b)
+      end
+
+      def write_entry(key, value, expires_in)
+        data = value.is_a?(Integer) ? value.to_s : dump(value)
+        if expires_in
+          client.call("SET", key, data, "PX", (Float(expires_in) * 1000).round)
+        else
+          client.call("SET", key, data)
+        end
+        true
+      end
+
+      def delete_entry(key) = client.call("DEL", key).positive?
+
+      def build_client(url, pool_size)
+        require "redis-client"
+        RedisClient.config(url: url).new_pool(size: pool_size, timeout: 1.0)
+      rescue LoadError
+        raise ConfigurationError, 'the Redis cache store needs `gem "redis-client"` in the Gemfile'
+      end
+    end
+  end
+end

data/lib/gemstack/cache/store.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module GemStack
+  module Cache
+    # The store interface. Subclasses implement read_entry / write_entry /
+    # delete_entry / clear (and may override increment for atomicity).
+    #
+    # Keys may be strings, symbols, arrays of parts, or objects with
+    # #cache_key (GemStack models have one); they are namespaced and hashed
+    # when longer than MAX_KEY bytes. Values must be Marshal-able.
+    class Store
+      MAX_KEY = 200
+      MISSING = Object.new.freeze
+
+      attr_reader :namespace, :default_expires_in
+
+      def initialize(namespace: nil, default_expires_in: nil)
+        @namespace = namespace&.to_s
+        @default_expires_in = default_expires_in
+      end
+
+      # Returns the cached value, or computes it with the block, stores and
+      # returns it. nil results are cached too. force: true recomputes.
+      def fetch(key, expires_in: default_expires_in, force: false)
+        normalized = normalize(key)
+        unless force
+          value = read_entry(normalized)
+          return value unless value.equal?(MISSING)
+        end
+        raise ArgumentError, "fetch needs a block on a cache miss" unless block_given?
+
+        value = yield
+        write_entry(normalized, value, expires_in)
+        value
+      end
+
+      def read(key)
+        value = read_entry(normalize(key))
+        value.equal?(MISSING) ? nil : value
+      end
+
+      def write(key, value, expires_in: default_expires_in)
+        write_entry(normalize(key), value, expires_in)
+        value
+      end
+
+      def exist?(key) = !read_entry(normalize(key)).equal?(MISSING)
+      def delete(key) = delete_entry(normalize(key))
+
+      # Not atomic in the base class; stores override it where they can.
+      def increment(key, by = 1, expires_in: default_expires_in)
+        normalized = normalize(key)
+        current = read_entry(normalized)
+        value = (current.equal?(MISSING) ? 0 : Integer(current)) + by
+        write_entry(normalized, value, expires_in)
+        value
+      end
+
+      def decrement(key, by = 1, **) = increment(key, -by, **)
+
+      def normalize(key)
+        raw = key_part(key)
+        full = namespace ? "#{namespace}:#{raw}" : raw
+        full.bytesize > MAX_KEY ? "#{full[0, 100]}:sha256:#{Digest::SHA256.hexdigest(full)}" : full
+      end
+
+      private
+
+      def key_part(key)
+        case key
+        when Array then key.map { |part| key_part(part) }.join("/")
+        when String, Symbol, Numeric then key.to_s
+        else key.respond_to?(:cache_key) ? key.cache_key.to_s : key.to_s
+        end
+      end
+
+      def dump(value) = Marshal.dump(value)
+      def load(data) = Marshal.load(data) # rubocop:disable Security/MarshalLoad -- only data this process wrote
+
+      def expires_at(expires_in)
+        expires_in ? monotonic + Float(expires_in) : nil
+      end
+
+      def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/gemstack/cache.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "gemstack/core"
+
+module GemStack
+  # Application caching:
+  #
+  #   GemStack.cache.fetch("product:#{id}", expires_in: 300) { Product.find(id) }
+  #   GemStack.cache.fetch([:stats, Date.today]) { expensive_report }   # array keys
+  #   GemStack.cache.increment("signups")
+  #   GemStack.cache.delete("product:#{id}")
+  #
+  # Stores are swappable (config.cache.store): :memory (default), :null
+  # (default in test), :redis, or any object implementing the Store interface.
+  # Redis is never required.
+  module Cache
+    class Config < Settings
+      setting :store, default: -> { GemStack.env.test? ? :null : :memory }
+      # Prefix for every key ("shop:product:1"). Required for Redis#clear.
+      setting :namespace, default: -> { GemStack.config.name }
+      # Seconds; nil = no expiry unless given per call.
+      setting :default_expires_in, default: nil
+      # MemoryStore: maximum entries per process before the least recently used are evicted.
+      setting :max_entries, default: 10_000
+      setting :redis_url, default: -> { ENV.fetch("REDIS_URL", "redis://localhost:6379/0") }
+      setting :redis_pool_size, default: -> { Integer(ENV.fetch("GEMSTACK_MAX_THREADS", 5)) }
+    end
+
+    class << self
+      def build(setting = GemStack.config.cache.store, config: GemStack.config.cache)
+        options = { namespace: config.namespace, default_expires_in: config.default_expires_in }
+        case setting
+        when :memory, "memory" then MemoryStore.new(max_entries: config.max_entries, **options)
+        when :null, "null" then NullStore.new(**options)
+        when :redis, "redis"
+          RedisStore.new(url: config.redis_url, pool_size: config.redis_pool_size, **options)
+        when Store then setting
+        else
+          raise ConfigurationError, "unknown cache store #{setting.inspect}" unless setting.respond_to?(:fetch)
+
+          setting
+        end
+      end
+    end
+  end
+
+  class << self
+    def cache
+      @cache ||= Cache.build
+    end
+
+    attr_writer :cache
+  end
+end
+
+require_relative "cache/store"
+require_relative "cache/memory_store"
+require_relative "cache/null_store"
+require_relative "cache/redis_store"
+
+GemStack::Config.namespace(:cache, GemStack::Cache::Config)

data/lib/gemstack/cli/add_generator.rb

@@ -0,0 +1,258 @@
+# frozen_string_literal: true
+
+module GemStack
+  class CLI < Thor
+    # `gemstack add FEATURE` — opt-in modules, so apps that don't use them
+    # pay nothing: realtime, auth, storage.
+    class AddGenerator < Generator
+      FEATURES = %w[realtime auth storage].freeze
+
+      AUTH_ROUTES = <<~RUBY
+        # Authentication (gemstack add auth) — docs/authentication.md
+        post "/auth/signup", to: "registrations#create"
+        post "/auth/login", to: "sessions#create"
+        get "/auth/me", to: "sessions#show"
+        delete "/auth/logout", to: "sessions#destroy"
+        post "/auth/password/forgot", to: "password_resets#create"
+        post "/auth/password/reset", to: "password_resets#update"
+        post "/auth/email/resend", to: "email_verifications#create"
+        post "/auth/email/verify", to: "email_verifications#update"
+        get "/auth/tokens", to: "api_tokens#index"
+        post "/auth/tokens", to: "api_tokens#create"
+        delete "/auth/tokens/:id", to: "api_tokens#destroy"
+      RUBY
+
+      MAIL_ENV = <<~ENV
+
+        # Email. In development messages are saved to tmp/mail instead of being sent.
+        # SMTP_URL=smtp://user:password@smtp.example.com:587
+        # MAIL_FROM="My App <hello@example.com>"
+        # APP_URL=https://example.com   # the frontend, for links in emails (default http://localhost:3000)
+      ENV
+
+      def initialize(feature, root:, output: $stdout, install: true)
+        super(output: output)
+        unless FEATURES.include?(feature)
+          raise Thor::Error,
+                "Unknown feature #{feature.inspect}. Available: #{FEATURES.join(", ")}"
+        end
+
+        @feature = feature
+        @root = root
+        @install = install
+      end
+
+      def run
+        public_send(:"add_#{@feature}")
+        self
+      end
+
+      def add_realtime
+        add_gem("gemstack-realtime")
+        template_files("realtime", override_root: @root).each do |rel, source|
+          next if rel.start_with?("frontend/") && !File.directory?(File.join(@root, "frontend"))
+
+          write(File.join(@root, rel), File.read(source))
+        end
+        add_test_support('require "gemstack/realtime/testing"', "GemStack::Realtime::Testing")
+        bundle_install
+      end
+
+      STORAGE_ENV = <<~ENV
+
+        # File storage. Development and tests store files under storage/ (git-ignored).
+        # STORAGE_SERVICE=s3            # production default
+        # S3_BUCKET=my-app-uploads
+        # AWS_REGION=eu-west-3
+        # AWS_ACCESS_KEY_ID=...  AWS_SECRET_ACCESS_KEY=...   (or an instance role)
+        # S3_ENDPOINT=https://<account>.r2.cloudflarestorage.com   # S3-compatible services
+      ENV
+
+      def add_storage
+        enable_module("storage")
+        ensure_base_classes(@root, :serializer)
+        copy_feature_templates("storage")
+        add_routes(%(post "/uploads", to: "uploads#create" # direct uploads (gemstack add storage)\n),
+                   marker: "uploads#create")
+        append_env_example(STORAGE_ENV, marker: "STORAGE_SERVICE")
+        append_line(".gitignore", "/storage/")
+        add_test_support('require "gemstack/storage/testing"', "GemStack::Storage::Testing")
+        status("note", "Gemfile", "add gem \"aws-sdk-s3\" for S3 in production (docs/storage.md)")
+        bundle_install
+      end
+
+      # For templates: whether `gemstack add auth` ran.
+      def auth? = Generator.uses?(@root, :auth)
+
+      def add_auth
+        require_database!("auth")
+        user_model = File.join(@root, "app/models/user.rb")
+        if File.file?(user_model) && !File.read(user_model).include?("GemStack::Auth::User")
+          raise Thor::Error, "app/models/user.rb already exists. Add `include GemStack::Auth::User` to it and " \
+                             "see docs/authentication.md (\"Existing users table\") instead."
+        end
+
+        enable_module("jobs")
+        enable_module("mail")
+        add_gem("gemstack-auth")
+        JobGenerator.install_migration(@root, output: @output) # emails are sent from background jobs
+        ensure_base_classes(@root, :model, :serializer, :mailer)
+        copy_feature_templates("auth", skip_migration: "*_create_auth_tables.rb")
+        include_in_application_controller("GemStack::Auth::Controller")
+        add_routes(AUTH_ROUTES, marker: "/auth/login")
+        append_env_example(MAIL_ENV, marker: "SMTP_URL")
+        add_test_support('require "gemstack/mail/testing"', "GemStack::Mail::Testing")
+        add_test_support('require "gemstack/auth/testing"', "GemStack::Auth::Testing")
+        bundle_install
+      end
+
+      private
+
+      def frontend? = File.directory?(File.join(@root, "frontend"))
+
+      def require_database!(feature)
+        return if Generator.uses?(@root, :db)
+
+        raise Thor::Error, "#{feature} stores its data in the database, and this app was created without one: " \
+                           "add `require \"gemstack/db\"` to config/app.rb, a config/database.yml and a driver gem " \
+                           "first (docs/database.md)"
+      end
+
+      # Writes a feature's templates (frontend/ files only when the app has
+      # one). %timestamp% in a path becomes a fresh migration version, unless a
+      # migration matching skip_migration already exists.
+      def copy_feature_templates(name, skip_migration: nil)
+        has_migration = skip_migration && !Dir.glob(File.join(@root, "db/migrations", skip_migration)).empty?
+        template_files(name, override_root: @root).sort.each do |rel, source|
+          next if rel.start_with?("frontend/") && !frontend?
+
+          if rel.include?("%timestamp%")
+            next status("identical", rel, "migration already present") if has_migration
+
+            rel = rel.sub("%timestamp%", Generator.migration_timestamp(@root))
+          end
+          content = File.read(source)
+          content = render(content, source) if rel.end_with?(".tt")
+          write(File.join(@root, rel.delete_suffix(".tt")), content)
+        end
+      end
+
+      def include_in_application_controller(mixin)
+        path = File.join(@root, "app/controllers/application_controller.rb")
+        return status("skip", path, "not found — add `include #{mixin}` yourself") unless File.file?(path)
+
+        content = File.read(path)
+        return status("identical", path, mixin) if content.include?(mixin)
+
+        updated = content.sub(/^class ApplicationController < .*\n/) { |line| "#{line}  include #{mixin}\n" }
+        return status("skip", path, "add `include #{mixin}` yourself") if updated == content
+
+        File.write(path, updated)
+        status("update", path, mixin)
+      end
+
+      def add_routes(block, marker:)
+        path = File.join(@root, "config/routes.rb")
+        content = File.file?(path) ? File.read(path) : ""
+        return status("identical", path, "routes") if content.include?(marker)
+        return status("skip", path, "no `GemStack.routes do` block — add the routes yourself") unless
+          content.match?(ControllerGenerator::ROUTES_BLOCK)
+
+        File.write(path, content.sub(ControllerGenerator::ROUTES_BLOCK) { |open| open + block.gsub(/^/, "  ") })
+        status("route", path)
+      end
+
+      def append_line(file, line)
+        path = File.join(@root, file)
+        return unless File.file?(path)
+
+        content = File.read(path)
+        return if content.lines.map(&:strip).include?(line)
+
+        File.write(path, "#{content.rstrip}\n#{line}\n")
+        status("update", path, line)
+      end
+
+      def append_env_example(text, marker:)
+        path = File.join(@root, ".env.example")
+        return unless File.file?(path)
+
+        content = File.read(path)
+        return if content.include?(marker)
+
+        File.write(path, "#{content.rstrip}\n#{text}")
+        status("update", path)
+      end
+
+      # Switches a module on in config/app.rb: uncomments its require line, or
+      # adds one after `require "gemstack"`. Apps from before 0.3.0 that list
+      # the module's gem in the Gemfile already have it.
+      def enable_module(name)
+        path = File.join(@root, "config/app.rb")
+        return status("identical", path, "gemstack/#{name}") if Generator.uses?(@root, name)
+
+        content = File.file?(path) ? File.read(path) : ""
+        line = %(require "gemstack/#{name}")
+        updated = content.sub(%r{^#\s*require "gemstack/#{name}".*$}, line)
+        updated = content.sub(/^require "gemstack"\n/) { |gemstack| "#{gemstack}#{line}\n" } if updated == content
+        return status("skip", path, "add `#{line}` to config/app.rb yourself") if updated == content
+
+        File.write(path, updated)
+        status("update", path, line)
+      end
+
+      # Adds the gem inside the GemStack `path` block (checkout apps) or as a
+      # versioned dependency next to gem "gemstack".
+      def add_gem(name)
+        path = File.join(@root, "Gemfile")
+        content = File.read(path)
+        return status("identical", path, name) if content.match?(/^\s*gem "#{name}"/)
+
+        updated = content.match?(/^path ".*" do\n/) ? add_to_path_block(content, name) : add_versioned(content, name)
+        if updated == content
+          raise Thor::Error,
+                "couldn't find where to add #{name} in the Gemfile; add `gem \"#{name}\"` yourself"
+        end
+
+        File.write(path, updated)
+        status("gemfile", path, name)
+      end
+
+      # Keeps the block's gems in alphabetical order (Bundler/OrderedGems).
+      def add_to_path_block(content, name)
+        content.sub(/^(path ".*" do\n)((?:  gem .*\n)*)/) do
+          opening = ::Regexp.last_match(1)
+          lines = ::Regexp.last_match(2) # before sort_by's regexes reset last_match
+          gems = (lines.lines + ["  gem \"#{name}\"\n"]).sort_by { |line| line[/"([^"]+)"/, 1] }
+          opening + gems.join
+        end
+      end
+
+      def add_versioned(content, name)
+        content.sub(/^gem "gemstack",.*\n/) { |line| "#{line}gem \"#{name}\", \"~> #{GemStack::VERSION}\"\n" }
+      end
+
+      def add_test_support(require_line, mixin)
+        path = File.join(@root, "test/test_helper.rb")
+        return unless File.file?(path)
+
+        content = File.read(path)
+        return status("identical", path) if content.include?(mixin)
+
+        content = content.sub(%(require "gemstack/testing"\n)) { |line| "#{line}#{require_line}\n" }
+        content = "#{content.rstrip}\nGemStack::TestCase.include #{mixin}\n"
+        File.write(path, content)
+        status("update", path, mixin)
+      end
+
+      def bundle_install
+        return unless @install
+
+        status("run", "bundle install")
+        install = -> { system("bundle", "install", "--quiet", chdir: @root) }
+        ok = defined?(Bundler) ? Bundler.with_unbundled_env(&install) : install.call
+        status("warning", "bundle install failed — run it yourself") unless ok
+      end
+    end
+  end
+end