itsi-server 0.1.19 → 0.1.20

data/lib/itsi/server/config/known_paths/wellknown-rfc5785.txt

@@ -0,0 +1,30 @@
+.well-known/
+.well-known/acme-challenge
+.well-known/apple-app-site-association
+.well-known/apple-developer-merchant-domain-association
+.well-known/ashrae
+.well-known/assetlinks.json
+.well-known/browserid
+.well-known/caldav
+.well-known/carddav
+.well-known/core
+.well-known/csvm
+.well-known/dnt
+.well-known/dnt-policy.txt
+.well-known/est
+.well-known/genid
+.well-known/hoba
+.well-known/host-meta
+.well-known/host-meta.json
+.well-known/keybase.txt
+.well-known/ni
+.well-known/openid-configuration
+.well-known/openorg
+.well-known/posh
+.well-known/reload-config
+.well-known/repute-template
+.well-known/stun-key
+.well-known/time
+.well-known/timezone
+.well-known/void
+.well-known/webfinger

data/lib/itsi/server/config/known_paths.rb

@@ -0,0 +1,17 @@
+module Itsi
+  class Server
+    module KnownPaths
+      Dir.glob(File.join(__dir__, 'known_paths', '**', '*.txt')).each do |file|
+        method_name = file[/known_paths\/(.*?)\.txt/,1].gsub(/([a-z])([A-Z])/, "\\1_\\2")
+          .gsub(/-|\.|\//, "_")
+          .gsub(/(^|\/)[0-9]/){|match| "FO"}.downcase
+        self.define_singleton_method(method_name) do
+          File.readlines(file).map do |s|
+            s.force_encoding('UTF-8')
+            s.valid_encoding? ? s.strip : s.encode('UTF-8', invalid: :replace, undef: :replace, replace: '').strip
+          end
+        end
+      end
+    end
+  end
+end

data/lib/itsi/server/config/middleware/_index.md

@@ -0,0 +1,54 @@
+---
+title: Middleware
+type: docs
+next: /features/server/lib/itsi/server/config/options/workers/
+url: /middleware
+cascade:
+  type: docs
+weight: 2
+---
+
+
+Itsi Middleware stacks are modular in nature.
+You can pick and choose **just** the features that make sense for you,
+and apply these on a *location-by-location* basis.
+
+{{% details title="What's a location?" closed="false" %}}
+
+> A location in Itsi is similar to a Location in NGINX. It's a logical container for all requests matching some combination of:
+* Routes/Route expressions
+* Request Methods
+* Content Types
+* Accept Headers
+* File types
+* Host/port/scheme.
+
+E.g.
+
+```ruby
+location "/admin/*" do
+
+  etag \
+    type: 'strong',
+    algorithm: 'md5',
+    min_body_size: 1024 * 1024
+  # ...
+
+  location "/public/images", extensions: %w[jpg png] do
+    compress \
+      min_size: 1024 * 1024,
+      level: 'fastest',
+      algorithms: %w[zstd gzip brotli deflate],
+      mime_types: %w[all],
+      compress_streams: true
+    # ...
+  end
+end
+```
+
+
+
+When a route matches a location block, it recursively inherits *all* middleware that is defined within outer ancestor blocks.
+Where a child and an ancestor define the same middleware, the child's middleware takes precedence.
+
+{{% /details %}}

data/lib/itsi/server/config/middleware/log_requests.md

@@ -0,0 +1,63 @@
+---
+title: Request Logging Middleware
+url: /middleware/log_requests
+weight: 1
+---
+
+The request logging middleware allows you to define customized log statements to occur before and/or after each request is processed.
+
+You can provide a log level and format string to be written before and after each request.
+
+
+```ruby {filename=Itsi.rb}
+    log_requests \
+      before: {
+        level: "INFO",
+        format: "[{request_id}] {method} {path_and_query} - {addr} "
+      },
+      after: {
+        level: "INFO",
+        format: "[{request_id}] └─ {status} in {response_time}"
+      }
+```
+
+
+The log statement can populated with several different placeholders.
+Available values are:
+
+### `before` Format String
+* `request_id` -  (A short, unique hexadecimal request identifier)
+* `request_id_full` - (A full 128-bit unique request identifier)
+* `method` - The HTTP method
+* `path` - The HTTP Path
+* `addr` - The client's IP address
+* `host` - The request host
+* `path_and_query` - The path and query combined
+* `query` - The request query string
+* `port` - The bound port
+* `start_time` - The request start time
+
+### `after` Format String
+* `request_id` - (A short, unique hexadecimal request identifier)
+* `request_id_full` - (A full 128-bit unique request identifier)
+* `status` - The HTTP status code
+* `addr` - The client's IP address
+* `response_time` - The response time in milliseconds
+
+
+### Path Attributes
+In addition to this, any capture groups referenced by container location blocks
+are also made available, to be interpolated into the log statement. E.g.:
+
+```ruby {filename=Itsi.rb}
+
+location "/users/:user_id" do # 1. If we capture user_id here.
+
+  log_requests before: {
+    level: "INFO",
+    format: "[{request_id}] User: {user_id}" # 2. Then we can log the user_id here.
+  }
+
+end
+
+```

data/lib/itsi/server/config/middleware/log_requests.rb

@@ -0,0 +1,33 @@
+module Itsi
+  class Server
+    module Config
+      class LogRequests < Middleware
+
+        attr_accessor :log_requests
+
+        insert_text <<~SNIPPET
+        log_requests \\
+          before: { level: ${1|"INFO","WARN","ERROR","DEBUG"|} format: ${2|"[{request_id}] {method} {path_and_query} - {addr} "|} },
+          after: { level: ${3|"INFO","WARN","ERROR","DEBUG"|}, format: ${4|"[{request_id}] └─ {status} in {response_time}"|} }
+        SNIPPET
+
+        detail "Enable logging before or after requests"
+
+        LogRequestConfig = TypedStruct.new do
+          {
+            level: (Required() + Enum(%w[INFO WARN ERROR DEBUG])).default("INFO"),
+            format: Type(String)
+          }
+        end
+
+        schema do
+          {
+            before: Type(LogRequestConfig),
+            after: Type(LogRequestConfig)
+          }
+        end
+
+      end
+    end
+  end
+end

data/lib/itsi/server/config/middleware.rb

@@ -0,0 +1,9 @@
+module Itsi
+  class Server
+    module Config
+      class Middleware
+        include ConfigHelpers
+      end
+    end
+  end
+end

data/lib/itsi/server/config/option.rb

@@ -0,0 +1,9 @@
+module Itsi
+  class Server
+    module Config
+      class Option
+        include ConfigHelpers
+      end
+    end
+  end
+end

data/lib/itsi/server/config/options/_index.md

@@ -0,0 +1,36 @@
+---
+title: Options
+type: docs
+next: /options/server/lib/itsi/server/config/options/workers/
+url: /options
+cascade:
+  type: docs
+weight: 1
+---
+
+Most of Itsi's capabilities are unlocked via the Itsi.rb config file.
+The config file uses a simple DSL, where you can write plain Ruby to define your application's configuration.
+For the best development experience, be sure to use [RubyLSP](https://shopify.github.io/ruby-lsp/) for snippets, autocomplete and documentation, right in your editor.
+
+{{< details title="An example Itsi.rb file" >}}
+
+
+```ruby {filename="Itsi.rb"}
+workers 2
+threads 2
+scheduler_threads 3
+
+fiber_scheduler true
+
+rate_limiter requests: 100, seconds: 10
+
+auth_basic realm: "Restricted Area", credentials_file: "credentials.txt"
+
+location "/app" do
+  get "/" do |req|
+    req.ok "Hello, World!"
+  end
+end
+
+```
+{{< /details >}}

data/lib/itsi/server/config/options/fiber_scheduler.md

@@ -0,0 +1,35 @@
+---
+title: Fiber Scheduler
+url: /options/fiber_scheduler
+weight: 3
+---
+Itsi supports processing requests in threads that are managed by a fiber scheduler.
+This allows Itsi to process a very large number of IO heavy requests concurrently without the memory and context switching overhead of managing multiple threads.
+
+Enabling Fiber Scheduler mode can drastically improve application performance if you perform large amounts of blocking IO operations.
+
+
+## Configuration File
+```ruby {filename="Itsi.rb"}
+# Enable Itsi's fiber scheduler mode
+# (This will use an instance of `Itsi::Scheduler`
+# This is Itsi's built in Fiber scheduler.)
+fiber_scheduler true
+```
+
+```ruby {filename="Itsi.rb"}
+# In the spirit of the Fiber::Scheduler interface,
+# you can bring your own scheduler!.
+# E.g. using the scheduler from the popular Async library.
+fiber_scheduler "Async::Scheduler"
+```
+
+{{< callout type="warning" >}}
+Running in Fiber scheduler mode can be a huge performance boon, but it's not without tradeoffs. Because it enables drastically more in-flight requests,
+it can have a substantial impact on memory usage. Similarly, it can increase the amount of simultaneous demand on pooled resources (like database connections or network sockets)
+and can cause increased contention on shared locks.
+
+While well-tuned Fiber based servers can drastically outperform their blocking counterparts in some scenarios, the above compromises can make it an unsafe blanket choice, particularly for some large applications with dependencies not specifically designed for a cooperative multitasking environment.
+
+To see Itsi's recommended approach to enjoying the benefits of a Fiber scheduler while managing these risks, consider using Itsi's [hybrid mode](https://itsi.fyi/options/scheduler_threads).
+{{< /callout >}}

data/lib/itsi/server/config/options/fiber_scheduler.rb

@@ -0,0 +1,18 @@
+module Itsi
+  class Server
+    module Config
+      class FiberScheduler < Option
+
+
+        insert_text "fiber_scheduler ${1|true,'Itsi::Scheduler'|}"
+
+        detail "Enable Fiber Scheduler mode"
+
+        schema do
+          Or(Bool(), (Type(String) + Required()))
+        end
+
+      end
+    end
+  end
+end

data/lib/itsi/server/config/options/threads.md

@@ -0,0 +1,39 @@
+---
+title: Threads
+url: /options/threads
+weight: 2
+---
+Itsi supports running in threaded mode. Spawning more than one thread per worker is helpful for applications that require high concurrency and low latency.
+
+Note, that while threading allows for concurrent execution of tasks, it also introduces additional overhead due to context switching between threads.
+This overhead can be significant, especially when dealing with a large number of concurrent tasks.
+To minimize this overhead, it's recommended to keep the number of threads low, and tune this parameter based on your specific use case.
+
+When running Itsi in blocking mode, the total number of concurrent requests will be equal to `workers` x `threads`
+
+## Configuration File
+The number of threads to use can be specified inside the configuration file (usually `./Itsi.rb` at the project root)
+using the `threads` function.
+
+## Examples
+```ruby {filename="Itsi.rb"}
+# Starts each worker with a single thread
+threads 1
+```
+
+```ruby {filename="Itsi.rb"}
+# Each worker will start 4 threads
+threads 4
+```
+
+Threads increase concurrency at the expense of memory usage and CPU overhead.
+For IO heavy workloads, consider using non-blocking mode instead, which can achieve higher concurrency with fewer threads.
+Non-blocking mode and threads can be used simultaneously in a hybrid configuration.
+
+## Command Line
+You can also override the number of threads using either the `-t` or `--threads` command line option.
+E.g.
+
+```bash
+itsi -t 3
+```

data/lib/itsi/server/config/options/threads.rb

@@ -0,0 +1,17 @@
+module Itsi
+  class Server
+    module Config
+      class Threads < Option
+
+        insert_text "threads ${1|1,2,Etc.nprocessors|}"
+
+        detail "Number of threads to run per worker"
+
+        schema do
+          Range(1..255)
+        end
+
+      end
+    end
+  end
+end

data/lib/itsi/server/config/options/workers.md

@@ -0,0 +1,43 @@
+---
+title: Workers
+url: /options/workers
+prev: /options
+weight: 1
+---
+Itsi is a preforking server. It can run as either a single process (`workers 1`) or in clustered mode (`workers > 1`).
+<br/>You can switch between the two without downtime.
+
+## Configuration File
+The number of workers to use can be specified inside the configuration file (usually `./Itsi.rb` at the project root)
+using the `workers` function.
+
+## Examples
+
+```ruby {filename="Itsi.rb"}
+# Starts a single worker, putting Itsi in single-process mode
+workers 1
+```
+
+```ruby {filename="Itsi.rb"}
+# Starts 4 workers, putting Itsi in cluster-process mode
+workers 4
+```
+
+```ruby {filename="Itsi.rb"}
+# Sets the number of workers to the number of CPU cores available on the system
+workers Etc.nprocessors
+```
+
+To maximize performance, it's typical to increase number of workers
+to be at or near the number of CPU cores available on your system.
+
+However, note hyperthreads and efficiency cores may impact this and using every available core is not
+always the most effective choice.
+
+## Command Line
+You can also override the number of workers using either the `-w` or workers `--workers` command line option.
+E.g.
+
+```bash
+itsi -w 4
+```

data/lib/itsi/server/config/options/workers.rb

@@ -0,0 +1,17 @@
+module Itsi
+  class Server
+    module Config
+      class Workers < Option
+
+        insert_text "workers ${1|1,2,Etc.nprocessors|}"
+
+        detail "Number of worker processes to run"
+
+        schema do
+          Range(1..255)
+        end
+
+      end
+    end
+  end
+end

data/lib/itsi/server/config/typed_struct.rb

@@ -0,0 +1,203 @@
+require 'date'
+require 'time'
+
+module Itsi
+  class Server
+    module Config
+      module TypedStruct
+        VALUE = 0
+        VALIDATE = 1
+
+        def self.new(defaults = nil, &defaults_blk)
+          defaults = TypedStruct.module_eval(&defaults_blk) if defaults_blk
+          unless defaults.is_a?(Hash)
+            return defaults
+          end
+          defaults.transform_values! { _1.is_a?(Validation) ? _1.default(nil) : _1 }
+          Struct.new(*defaults.keys, keyword_init: true) do
+            define_method(:initialize) do |*input, validate: true, **raw_input|
+              raw_input.transform_keys!{|k| k.to_s.downcase.to_sym }
+              raw_input.merge!(input.pop) if input.last.is_a?(Hash)
+
+              excess_keys = raw_input.keys - defaults.keys
+
+              raise "Unsupported keys #{excess_keys}" if excess_keys.any?
+
+              initial_values = defaults.each_with_object({}) do |(k, default_config), inputs|
+                value = raw_input.key?(k) ? raw_input[k] : default_config[VALUE].dup
+                next inputs[k] = value unless validate
+
+                begin
+                  inputs[k] = default_config[VALIDATE].validate!(value)
+                rescue StandardError => e
+                  raise ArgumentError, e.message
+                end
+              end
+
+              super(initial_values)
+              validator = self.class.instance_eval { @validators }
+              instance_eval(&validator) if validator
+            end
+
+            define_singleton_method(:validate) do |&blk|
+              @validators = blk
+            end
+
+            defaults.each do |key, (_, validation)|
+              define_method(:"#{key}?") do
+                !!self[key]
+              end
+              define_method(:"#{key}=") do |value|
+                self[key] = validation.validate!(value)
+              end
+            end
+
+            define_method(:[]=) do |key, value|
+              super(key, defaults[key][VALIDATE].validate!(value))
+            end
+
+            def is_typed_struct?
+              true
+            end
+
+            def merge_config(other)
+              self.class.new(members.map do |key|
+                value = self[key]
+                has_merged_val = (other.is_a?(Hash) ? other.key?(key) : other.member?(key))
+                next [key, value] unless has_merged_val
+
+                [
+                  key,
+                  TypedStruct.typed_struct?(value) ? value.merge_config(other[key]) : other[key]
+                ]
+              end.to_h)
+            end
+
+            def to_h
+              super.transform_values { |v| v.respond_to?(:merge_config) ? v.to_h : v }
+            end
+          end
+        end
+
+        def self.typed_struct?(inst)
+          inst.respond_to?(:is_typed_struct?) && inst.is_typed_struct?
+        end
+
+        class Validation
+          attr_reader :name, :validations
+          attr_accessor :next
+
+          def initialize(name, validations)
+            @name = name.to_s
+            @validations = Array(validations)
+            @next = nil
+          end
+
+          def inspect
+            "#{@name}"
+          end
+
+          def +(other)
+            tail = self
+            while tail.next
+              tail = tail.next
+            end
+            tail.next = other
+            self
+          end
+
+          def default(value)
+            [value, self]
+          end
+
+          def validate!(value)
+            value = value.to_s if value.is_a?(Symbol)
+            @validations.each do |validation|
+              case validation
+              when Proc
+                unless validation.call(value)
+                  raise ArgumentError,
+                        "─  `#{@name}` validation failed. Invalid value: #{value.inspect}"
+                end
+              when Array
+                unless !value || validation.include?(value)
+                  raise ArgumentError,
+                        "─  `#{@name}` validation failed. Invalid #{validation} value: #{value.inspect}"
+                end
+              when Range
+                unless !value || validation.include?(value)
+                  raise ArgumentError,
+                        "─  `#{@name}` validation failed. Invalid #{validation} value: #{value.inspect}"
+                end
+              when Regexp
+                unless !value || validation.match?(value)
+                  raise ArgumentError,
+                        "─  `#{@name}` validation failed. Invalid #{validation} value: #{value.inspect}"
+                end
+              when Class
+
+                if value && !value.is_a?(validation)
+                  begin
+                    value = \
+                      if validation.eql?(Time) then Time.parse(value.to_s)
+                      elsif validation.eql?(::Date) then Date.parse(value.to_s)
+                      elsif validation.eql?(Float) then Float(value)
+                      elsif validation.eql?(String) then value.to_s
+                      elsif validation.eql?(Symbol) then value.is_a?(Symbol) ? value : value.to_s.to_sym
+                      else validation.new(value)
+                      end
+                  rescue StandardError => e
+                    raise ArgumentError,
+                          "─ `#{@name}` Validation Failed. Invalid #{validation.to_s.split("::").last} value: #{value.inspect}. Failure reason: \n └#{e.message}"
+                  end
+                end
+              end
+            end
+            if self.next
+              self.next.validate!(value)
+            else
+              value
+            end
+          end
+        end
+
+        {
+          Bool: Validation.new(:Bool, [[true, false]]),
+          Required: Validation.new(:Required, ->(value) { !value.nil? }),
+          Or: ->(validation_a, validation_b){
+            Validation.new(:Or, ->(v){
+              validation_a.validate!(v) || validation_b.validate!(v)
+            })
+          },
+          Range: ->(input_range) {
+            Validation.new(:Range, [input_range])
+          },
+          Length: lambda { |input_length|
+            Validation.new(:Length, ->(value) { input_length === value.length })
+          },
+          Type: ->(input_type) { Validation.new(:Type, input_type) },
+          Enum: ->(allowed_values) { Validation.new(:Enum, [allowed_values.map{|v| v.kind_of?(Symbol) ? v.to_s : v}]) },
+          Array: lambda { |*value_validations|
+            Validation.new(:Array, [::Array, lambda { |value|
+              return true unless value
+
+              value&.map! do |v|
+                value_validations.all? { v = _1.validate!(v) }
+                v
+              end
+            }])
+          }
+        }.each do |name, factory|
+          if factory.is_a?(Proc)
+            define_singleton_method(name, &factory)
+            define_singleton_method(name.to_s.downcase, &factory)
+          else
+            const_set(name, factory)
+            define_singleton_method(name, -> { factory })
+            define_singleton_method(name.to_s.downcase, -> { factory })
+          end
+        end
+      end
+    end
+  end
+end