syntropy 0.30.0 → 0.32.0

data/lib/syntropy/app.rb

@@ -6,7 +6,7 @@ require 'yaml'
 require 'papercraft'
 
 require 'syntropy/errors'
-require 'syntropy/module'
+require 'syntropy/module_loader'
 require 'syntropy/routing_tree'
 require 'syntropy/mime_types'
 
@@ -35,6 +35,7 @@ module Syntropy
     end
 
     attr_reader :module_loader, :routing_tree, :root_dir, :mount_path, :env
+    attr_accessor :raise_on_internal_server_error
 
     def initialize(**env)
       @machine = env[:machine]
@@ -101,6 +102,21 @@ module Syntropy
       route
     end
 
+    def setup_db(db_path:, schema_root: '_schema')
+      @env[:db_path] = db_path
+      @env[:schema_root] = schema_root
+
+      class << self
+        def connection_pool
+          @connection_pool ||= DB::ConnectionPool.new(@machine, @env[:db_path], 4)
+        end
+
+        def schema
+          @schema ||= DB::Schema.new(module_loader: @module_loader, schema_root: @env[:schema_root])
+        end
+      end
+    end
+
     private
 
     # Handles a not found error, taking into account hooks up the tree from the
@@ -321,7 +337,7 @@ module Syntropy
           }
           body {
             markdown md
-            auto_refresh_watch! if @env[:dev_mode]
+            auto_refresh! if @env[:dev_mode]
           }
         }
       }
@@ -450,16 +466,28 @@ module Syntropy
     end
 
     RAW_DEFAULT_ERROR_HANDLER = ->(req, err) {
+      status = Syntropy::Error.http_status(err)
+
       msg = err.message
       msg = nil if msg.empty? || (req.method == 'head')
-      req.respond(msg, ':status' => Syntropy::Error.http_status(err)) rescue nil
+      req.respond(msg, ':status' => status) rescue nil
     }
 
-    def default_error_handler
+    RAISE_INTERNAL_SERVER_ERROR_DEFAULT_ERROR_HANDLER = ->(req, err) {
+      status = Syntropy::Error.http_status(err)
+      raise if status == HTTP::INTERNAL_SERVER_ERROR
 
+      msg = err.message
+      msg = nil if msg.empty? || (req.method == 'head')
+      req.respond(msg, ':status' => status) rescue nil
+    }
+
+    def default_error_handler
       @default_error_handler ||= begin
         if @builtin_applet
           @builtin_applet.module_loader.load('/default_error_handler')
+        elsif @raise_on_internal_server_error
+          RAISE_INTERNAL_SERVER_ERROR_DEFAULT_ERROR_HANDLER
         else
           RAW_DEFAULT_ERROR_HANDLER
         end
@@ -471,6 +499,8 @@ module Syntropy
     #
     # @return [void]
     def start
+      @module_loader.load('_setup', raise_on_missing: false)
+
       @machine.spin do
         # we do startup stuff asynchronously, in order to first let Syntropy do
         # its setup tasks.
@@ -489,9 +519,6 @@ module Syntropy
     #
     # @return [void]
     def file_watcher_loop
-      wf = @env[:watch_files]
-      period = wf.is_a?(Numeric) ? wf : 0.1
-
       @machine.file_watch(@root_dir, UM::IN_CREATE | UM::IN_DELETE | UM::IN_CLOSE_WRITE) { |e|
         fn = e[:fn]
         @logger&.info(message: 'File change detected', fn: fn)
@@ -499,8 +526,6 @@ module Syntropy
         debounce_file_change
       }
 
-
-
       # Syntropy.file_watch(@machine, @root_dir, period: period) do |event, fn|
       #   @logger&.info(message: 'File change detected', fn: fn)
       #   @module_loader.invalidate_fn(fn)

data/lib/syntropy/applets/builtin/default_error_handler.rb

@@ -23,7 +23,7 @@ ErrorPage = ->(error:, status:, backtrace:) {
           }
         end
       }
-      auto_refresh_watch!
+      auto_refresh!
     }
   }
 }
@@ -32,7 +32,7 @@ def transform_backtrace(backtrace)
   backtrace.map do
     if (m = it.match(/^(.+:\d+):/))
       location = m[1]
-      { entry: it, url: "vscode://file/#{location}" }
+      { entry: it, url: "zed://file/#{location}" }
     else
       { entry: it, url: nil }
     end
@@ -43,7 +43,7 @@ def error_response_html(req, error)
   status = Syntropy::Error.http_status(error)
   backtrace = transform_backtrace(error.backtrace)
   html = Papercraft.html(ErrorPage, error:, status:, backtrace:)
-  req.html_response(html, ':status' => status)
+  req.respond_html(html, ':status' => status)
 end
 
 def error_response_raw(req, error)

data/lib/syntropy/applets/builtin/req.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 export ->(req) {
-  req.json_response({
+  req.respond_json({
     headers: req.headers
   })
 }

data/lib/syntropy/db/connection_pool.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'extralite'
+
+module Syntropy
+  module DB
+    class ConnectionPool
+      attr_reader :count
+
+      def initialize(machine, fn, max_conn)
+        @machine = machine
+        @fn = fn
+        @count = 0
+        @max_conn = max_conn
+        @queue = UM::Queue.new
+        @key = :"connection_pool_#{object_id}"
+      end
+
+      def with_db
+        if (db = Thread.current[@key])
+          @machine.snooze
+          return yield(db)
+        end
+
+        db = checkout
+        begin
+          Thread.current[@key] = db
+          yield(db)
+        ensure
+          Thread.current[@key] = nil
+          checkin(db)
+        end
+      end
+
+      def query(sql, *, **, &)
+        with_db { it.query(sql, *, **, &) }
+      end
+
+      def execute(sql, *, **)
+        with_db { it.execute(sql, *, **) }
+      end
+
+      def close
+        while @queue.count > 0
+          db = @machine.shift(@queue)
+          db.close
+          @count -= 1
+        end
+      end
+
+      private
+
+      def checkout
+        return make_db_instance if @queue.count == 0 && @count < @max_conn
+
+        @machine.shift(@queue)
+      end
+
+      def checkin(db)
+        @machine.push(@queue, db)
+      end
+
+      def make_db_instance
+        Extralite::Database.new(@fn, wal: true).tap do
+          @count += 1
+          it.on_progress(mode: :at_least_once, period: 320, tick: 10) { @machine.snooze }
+        end
+      end
+    end
+  end
+end

data/lib/syntropy/db/schema.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Syntropy
+  module DB
+    class Schema
+      def initialize(module_loader: nil, schema_root: '_schema', &)
+        @migrations = {}
+        @module_loader = module_loader
+        @schema_root = schema_root
+        load_schema_from_modules if @module_loader
+        run_schema_block(&) if block_given?
+      end
+
+      def apply(connection_pool)
+        execute_migrations(connection_pool)
+      end
+
+      def current_version(connection_pool)
+        connection_pool.with_db do |db|
+          get_schema_version(db)
+        end
+      end
+
+      private
+
+      def load_schema_from_modules
+        modules = @module_loader.list(@schema_root)
+        modules.each do |name|
+          @migrations[File.basename(name)] = @module_loader.load(name)
+        end
+      end
+
+      class SchemaBlockRunner
+        def initialize(migrations, &)
+          @migrations = migrations
+          instance_eval(&)
+        end
+
+        def initial(&block)
+          @migrations['0000'] = block
+        end
+
+        def version(key, &block)
+          @migrations[key] = block
+        end
+      end
+
+      def run_schema_block(&)
+        SchemaBlockRunner.new(@migrations, &)
+      end
+
+      def execute_migrations(connection_pool)
+        connection_pool.with_db do |db|
+          current_version = get_schema_version(db)
+          migrations_keys = @migrations.keys.sort
+          migrations_keys.select { it > current_version } if current_version
+
+          migrations_keys.each do |key|
+            db.transaction do
+              @migrations[key].(db)
+              set_schema_version(db, key)
+              current_version = key
+            end
+          end
+
+          current_version
+        end
+      end
+
+      def get_schema_version(db)
+        db.execute <<~SQL
+          create table if not exists __syntropy_schema__(
+            k text primary key,
+            v text
+          );
+        SQL
+        db.query_single_splat <<~SQL
+          select v from __syntropy_schema__
+          where k = 'version'
+        SQL
+      end
+
+      def set_schema_version(db, version)
+        db.execute <<~SQL, v: version
+          insert into __syntropy_schema__ (k, v)
+          values ('version', :v)
+          on conflict(k) do update set v = :v
+        SQL
+      end
+    end
+  end
+end

data/lib/syntropy/db/store.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Syntropy
+  module DB
+    class Store
+      def initialize(connection_pool)
+        @connection_pool = connection_pool
+      end
+
+      def query(sql, *, **)
+        @connection_pool.with_db { it.query(sql, *, **) }
+      end
+
+      def query_single_row(sql, *, **)
+        @connection_pool.with_db { it.query_single(sql, *, **) }
+      end
+
+      def query_single_value(sql, *, **)
+        @connection_pool.with_db { it.query_single_splat(sql, *, **) }
+      end
+
+      def execute(sql, *, **)
+        @connection_pool.with_db { it.execute(sql, *, **) }
+      end
+
+      def transaction(&)
+        @connection_pool.with_db(&)
+      end
+    end
+  end
+end

data/lib/syntropy/dev_mode.rb

@@ -4,7 +4,7 @@ TAG_DEBUG_PROC = ->(level, fn, line, col) {
   {
     'data-syntropy-level' => level,
     'data-syntropy-fn'    => fn,
-    'data-syntropy-loc'   => "vscode://file/#{fn}:#{line}:#{col}"
+    'data-syntropy-loc'   => "zed://file/#{fn}:#{line}:#{col}"
   }
 }
 

data/lib/syntropy/errors.rb

@@ -84,4 +84,10 @@ module Syntropy
 
   class BadRequestError < Error
   end
+
+  class InvalidRequestContentTypeError < Error
+    def http_status
+      HTTP::UNSUPPORTED_MEDIA_TYPE
+    end
+  end
 end

data/lib/syntropy/http/client.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'syntropy/http/client_connection'
+require 'uri'
+
+module Syntropy
+  module HTTP
+    class Client
+      def initialize(machine)
+        @machine = machine
+      end
+
+      def get(url, **headers, &)
+        uri = URI.parse(url)
+        headers = headers.merge(
+          ':method' => 'GET',
+          ':path' => uri.request_uri
+        )
+        req(uri, **headers, &)
+      end
+
+      private
+
+      # @param uri [URI]
+      def req(uri, **headers)
+        connection = make_connection(uri.scheme, uri.host, uri.port)
+        response_headers = connection.req(**headers)
+        if block_given?
+          yield(response_headers, connection)
+        else
+          [response_headers, connection.get_response_body(response_headers)]
+        end
+      end
+
+      def make_connection(_scheme, host, port)
+        ip = (host =~ /^\d+\.\d+\.\d+\.\d+$/) ? host : @machine.resolve(host)[0]
+
+        fd = @machine.tcp_connect(ip, port)
+        Syntropy::HTTP::ClientConnection.new(@machine, fd)
+      end
+    end
+  end
+end

data/lib/syntropy/http/client_connection.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'syntropy/errors'
+require 'syntropy/http/io_extensions'
+
+module Syntropy
+  module HTTP
+    class ClientConnection
+      attr_reader :fd, :response_headers, :logger
+
+      def initialize(machine, fd, io_mode: :socket)
+        @machine = machine
+        @fd = fd
+        @io = machine.io(fd, io_mode)
+      end
+
+      def req(body: nil, **headers)
+        if body
+          headers = headers.merge(
+            'Content-Length' => body.bytesize
+          )
+        end
+        @io.http_write_request_headers(**headers)
+        if body
+          @io.write(body)
+        end
+
+        @io.http_read_response_headers
+      end
+
+      def get_response_body(headers)
+        @io.http_read_body(headers)
+      end
+    end
+  end
+end

data/lib/syntropy/http/io_extensions.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require 'syntropy/errors'
+
+module Syntropy
+  module HTTP
+    module ProtocolMethods
+      RE_REQUEST_LINE = /^(get|head|options|trace|put|delete|post|patch|connect)\s+([^\s]+)\s+HTTP\/([019\.]{1,3})/i
+      RE_RESPONSE_LINE = /^HTTP\/1\.1\s+(\d{3})(\s+.+)?$/i
+      RE_HEADER_LINE = /^([a-z0-9\-]+):\s+(.+)/i
+
+      MAX_REQUEST_LINE_LEN = 1 << 14 # 16KB
+      MAX_RESPONSE_LINE_LEN = 1 << 8 # 256
+      MAX_HEADER_LINE_LEN = 1 << 13 # 8KB
+      MAX_CHUNK_SIZE_LEN = 16
+
+      # @return [Hash] headers
+      def http_read_request_headers
+        line = read_line(MAX_REQUEST_LINE_LEN)
+        return nil if !line
+
+        m = line.match(RE_REQUEST_LINE)
+        raise ProtocolError, 'Invalid request line' if !m
+
+        http_version = m[3]
+        raise UnsupportedHTTPVersionError, 'HTTP version not supported' if http_version != '1.1'
+
+        headers = {
+          ':method'   => m[1].downcase,
+          ':path'     => m[2]
+        }
+
+        loop do
+          line = read_line(MAX_HEADER_LINE_LEN)
+          break if line.nil? || line.empty?
+
+          m = line.match(RE_HEADER_LINE)
+          raise ProtocolError, "Invalid header: #{line[0..2047].inspect}" if !m
+
+          headers[m[1].downcase] = m[2]
+        end
+
+        headers
+      end
+
+      def http_read_response_headers
+        line = read_line(MAX_RESPONSE_LINE_LEN)
+        return nil if !line
+
+        m = line.match(RE_RESPONSE_LINE)
+        raise ProtocolError, 'Invalid response line' if !m
+
+        headers = {
+          ':status'   => m[1].to_i
+        }
+
+        loop do
+          line = read_line(MAX_HEADER_LINE_LEN)
+          break if line.nil? || line.empty?
+
+          m = line.match(RE_HEADER_LINE)
+          raise ProtocolError, "Invalid header: #{line[0..2047].inspect}" if !m
+
+          k = m[1].downcase
+          if (h = headers[k])
+            (h = headers[k] = [h]) if !h.is_a?(Array)
+            h << m[2]
+          else
+            headers[k] = m[2]
+          end
+        end
+
+        headers
+      end
+
+      def http_read_body(headers)
+        content_length = headers['content-length']
+        if content_length
+          chunk = read(content_length.to_i)
+          return chunk
+        end
+
+        chunked_encoding = headers['transfer-encoding']&.downcase == 'chunked'
+        if chunked_encoding
+          buf = +''
+          while (chunk = http_read_cte_chunk(nil))
+            buf << chunk
+          end
+          return buf
+        end
+
+        nil
+      end
+
+      def http_skip_body(headers)
+        content_length = headers['content-length']
+        if content_length
+          return skip(content_length.to_i)
+        end
+
+        chunked_encoding = headers['transfer-encoding']&.downcase == 'chunked'
+        if chunked_encoding
+          while http_skip_cte_chunk
+          end
+        end
+
+        nil
+      end
+
+      def http_read_body_chunk(headers)
+        content_length = headers['content-length']
+        if content_length
+          chunk = read(content_length.to_i)
+          return chunk
+        end
+
+        chunked_encoding = headers['transfer-encoding']&.downcase == 'chunked'
+        return http_read_cte_chunk(nil) if chunked_encoding
+
+        nil
+      end
+
+      def http_write_request_headers(headers)
+        method = headers[':method'] || (raise BadRequestError)
+        path = headers[':path'] || (raise BadRequestError)
+
+        lines = ["#{method} #{path} HTTP/1.1\r\n"]
+        headers.each do |k, v|
+          next if k =~ /^\:/
+
+          if v.is_a?(Array)
+            v.each { lines << "#{k}: #{it}\r\n" }
+          else
+            lines << "#{k}: #{v}\r\n"
+          end
+        end
+        lines << "\r\n"
+        write(*lines)
+      end
+
+      private
+
+      def http_read_cte_chunk(buffer)
+        chunk_size_str = read_line(MAX_CHUNK_SIZE_LEN)
+        return nil if !chunk_size_str
+
+        chunk_size = chunk_size_str.to_i(16)
+        if chunk_size == 0
+          read_line(0)
+          return nil
+        end
+
+        chunk = read(chunk_size)
+        read_line(0)
+
+        buffer ? (buffer << chunk) : chunk
+      end
+
+      def http_skip_cte_chunk
+        chunk_size_str = read_line(MAX_CHUNK_SIZE_LEN)
+        return if !chunk_size_str
+
+        chunk_size = chunk_size_str.to_i(16)
+        if chunk_size == 0
+          read_line(0)
+          return nil
+        end
+
+        chunk = skip(chunk_size)
+        read_line(0)
+      end
+    end
+  end
+end
+
+UringMachine::IO.include(Syntropy::HTTP::ProtocolMethods)

data/lib/syntropy/http/server.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'syntropy/http/connection'
+require 'syntropy/http/server_connection'
 
 module Syntropy
   module HTTP
@@ -119,13 +119,13 @@ module Syntropy
       end
 
       def accept_incoming(listen_fd)
-        @machine.accept_each(listen_fd) { start_client_connection(it) }
+        @machine.accept_each(listen_fd) { start_connection(it) }
       rescue UM::Terminate
-        # terminated
+        @machine.shutdown(listen_fd, UM::SHUT_RD)
       end
 
-      def start_client_connection(fd)
-        conn = Connection.new(self, @machine, fd, @env, &@app)
+      def start_connection(fd)
+        conn = ServerConnection.new(@machine, fd, @env, &@app)
         f = @machine.spin(conn) do
           it.run
         ensure