syntropy 0.40.0 → 0.42.0

data/lib/syntropy/collection.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'json'
+require 'syntropy/markdown'
+
+module Syntropy
+  # A collection represents a collection of data entities represented in files.
+  class Collection
+    attr_reader :root, :root_dir, :url_base
+
+    # Initializes a collection.
+    #
+    # @param machine [UringMachine]
+    # @param root [String] collection root
+    # @param url_base [String] URL base
+    # @return [void]
+    def initialize(machine:, root:, url_base:)
+      @machine = machine
+      @root_dir = File.expand_path(root)
+      @url_base = url_base
+      @list = {} # list of non-directory items
+      @root = make_tree_root
+      @dirs = { @root_dir => @root } # mapping directory paths to tree items
+
+      calc_collection_tree
+    end
+
+    # Returns a list of items in the collection.
+    #
+    # @return [Array] items
+    def list
+      @list.values
+    end
+
+    # Returns the item corresponding to the given URL.
+    #
+    # @param url [String] URL
+    # @return [Hash] item
+    def get(url)
+      @list[url]
+    end
+
+    private
+
+    def make_tree_root
+      {
+        fn: find_directory_data_file(@root_dir),
+        ref: '',
+        url: url_base,
+        type: :directory,
+        items: []
+      }
+    end
+
+    # Calculates the collection tree.
+    #
+    # @return [void]
+    def calc_collection_tree
+      queue = UM::Queue.new
+      Dir[File.join(@root_dir, '**/*')].each do |fn|
+        next if File.basename(fn) =~ /^_/
+
+        ref = fn_to_ref(fn)
+        url = File.join(@url_base, ref)
+        parent_fn = File.expand_path(File.join(fn, '..'))
+        parent = @dirs[parent_fn]
+        raise Error, "Parent not found: #{parent_fn}" if !parent
+
+        if File.directory?(fn)
+          dir = {
+            fn: find_directory_data_file(fn),
+            ref:,
+            url:,
+            type: :directory,
+            items: []
+          }
+          @dirs[fn] = dir
+          parent[:items] << dir
+          @machine.push(queue, dir) if dir[:fn]
+        else
+          item = {
+            fn:,
+            ref:,
+            url:
+          }
+          @list[url] = item
+          parent[:items] << item
+          @machine.push(queue, item)
+        end
+      end
+      4.times { @machine.push(queue, :stop) }
+
+      fibers = 4.times.map {
+        @machine.spin {
+          loop {
+            item = @machine.shift(queue)
+            break if item == :stop
+
+            load_item(item)
+          }
+        }
+      }
+      @machine.join(fibers)
+
+      add_prev_next_links
+    end
+
+    def scan_files(path, queue)
+    end
+
+    # Converts a filename to a ref.
+    #
+    # @param fn [String]
+    # @return [String] ref
+    def fn_to_ref(fn)
+      @ref_rel_regexp ||= /^#{@root_dir}\/(.+)/
+      rel = fn.match(@ref_rel_regexp)[1]
+
+      (m = rel.match(/^(.+)\.(md|json|yml|yaml)$/)) ? m[1] : rel
+    end
+
+    # Returns the data file for the given directory.
+    #
+    # @param dir [String]
+    # @return [String, nil] directory data file
+    def find_directory_data_file(dir)
+      (
+        detect_file(File.join(dir, '_index.yml')) ||
+        detect_file(File.join(dir, '_index.json'))
+      )
+    end
+
+    # Detects if the given file exists.
+    #
+    # @param fn [String]
+    # @return [String, nil] filename if exists, nil otherwise
+    def detect_file(fn)
+      File.file?(fn) ? fn : nil
+    end
+
+    # Loads the given item.
+    #
+    # @param item [Hash]
+    # @return [void]
+    def load_item(item)
+      basename = File.basename(item[:fn])
+      case (ext = File.extname(basename))
+      when '.md'
+        item[:type] ||= :markdown
+        load_item_markdown(item)
+      when '.json'
+        item[:type] ||= :json
+        load_item_json(item)
+      when '.yml', '.yaml'
+        item[:type] ||= :yaml
+        load_item_yaml(item)
+      else
+        raise Syntropy::Error, "Unkown file type #{ext}"
+      end
+    end
+
+    # Loads ands parses the file for the given markdown item.
+    #
+    # @param item [Hash]
+    # @return [void]
+    def load_item_markdown(item)
+      attributes, body = Markdown.parse_file(item[:fn], {})
+      item.merge!(attributes:, body:)
+    end
+
+    # Loads ands parses the file for the given JSON item.
+    #
+    # @param item [Hash]
+    # @return [void]
+    def load_item_json(item)
+      data = @machine.file_read(item[:fn])
+      item[:data] = JSON.parse(data, symbolize_names: true)
+    end
+
+    YAML_OPTS = {
+      permitted_classes: [Date],
+      symbolize_names: true
+    }.freeze
+
+    # Loads ands parses the file for the given YAML item.
+    #
+    # @param item [Hash]
+    # @return [void]
+    def load_item_yaml(item)
+      data = @machine.file_read(item[:fn])
+      item[:data] = YAML.safe_load(data, **YAML_OPTS)
+    end
+
+    def add_prev_next_links
+      last = nil
+      @list.each_value { |item|
+        if last
+          item[:prev] = last
+          last[:next] = item
+        end
+        last = item
+      }
+    end
+  end
+end

data/lib/syntropy/connection_pool.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require 'extralite'
+
+module Syntropy
+  # ConnectionPool implements concurrent access to an SQLite database.
+  class ConnectionPool
+    attr_reader :count
+
+    # Initializes the connection pool.
+    #
+    # @param machine [UringMachine] machine instance
+    # @param fn [String] database filename
+    # @param max_conn [Integer] maximum number of connections
+    # @return [void]
+    def initialize(machine, fn, max_conn)
+      @machine = machine
+      @fn = fn
+      @count = 0
+      @max_conn = max_conn
+      @queue = UM::Queue.new
+      @key = :"connection_pool_#{object_id}"
+    end
+
+    # Checks out a database connection, passing it to the given block. This
+    # method is reentrant - called from the same fiber, it will yield the same
+    # database connection.
+    #
+    # @return [Extralite::Database] database connection
+    def with_db
+      if (db = Thread.current[@key])
+        @machine.snooze
+        return yield(db)
+      end
+
+      db = checkout
+      begin
+        Thread.current[@key] = db
+        yield(db)
+      ensure
+        Thread.current[@key] = nil
+        checkin(db)
+      end
+    end
+
+    # Performs the given query by acquiring a database connection and running
+    # the query.
+    #
+    # @return [Array<Hash>] result rows
+    def query(*, **, &)
+      with_db { it.query(*, **, &) }
+    end
+
+    # Performs the given query by acquiring a database connection and running
+    # the query.
+    #
+    # @return [<Hash>, nil] result row
+    def query_single_row(*, **)
+      with_db { it.query_single(*, **) }
+    end
+
+    # Performs the given query by acquiring a database connection and running
+    # the query.
+    #
+    # @return [any] result value
+    def query_single_value(*, **)
+      with_db { it.query_single_splat(*, **) }
+    end
+
+    # Executes the given query by acquiring a database connection and running
+    # the query.
+    #
+    # @return [Integer] number of changed rows
+    def execute(*, **)
+      with_db { it.execute(*, **) }
+    end
+
+    # Closes the connection pool by removing all connections.
+    #
+    # @return [void]
+    def close
+      while @queue.count > 0
+        db = @machine.shift(@queue)
+        db.close
+        @count -= 1
+      end
+    end
+
+    private
+
+    # Checks out a connection from the pool.
+    #
+    # @return [Extralite::Database] database connection
+    def checkout
+      return make_db_instance if @queue.count == 0 && @count < @max_conn
+
+      @machine.shift(@queue)
+    end
+
+    # Checks in a connection to the pool.
+    #
+    # @param db [Extralite::Database] database connection
+    # @return [void]
+    def checkin(db)
+      @machine.push(@queue, db)
+    end
+
+    # Creates a database connection.
+    #
+    # @return [Extralite::Database] database connection
+    def make_db_instance
+      Extralite::Database.new(@fn, wal: true).tap do
+        @count += 1
+        it.on_progress(mode: :at_least_once, period: 320, tick: 10) { @machine.snooze }
+      end
+    end
+  end
+end

data/lib/syntropy/controller_extensions.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'securerandom'
+require 'syntropy/collection'
 
 module Syntropy
   # Utilities for use in modules
@@ -113,5 +114,16 @@ module Syntropy
 
       send(sym, req)
     end
+
+    # Opens a collection with the given arguments.
+    #
+    # @param rel_root [String] relative collection root
+    # @param url_base [String] URL base
+    # @return [Syntropy::Collection] collection
+    def collection(rel_root, url_base:)
+      invalidate_on_file_change(File.join(rel_root, '**'))
+      path = normalize_import_ref(rel_root)
+      Collection.new(machine: @machine, root: File.join(@env[:app_root], path), url_base:)
+    end
   end
 end

data/lib/syntropy/http/server_connection.rb

@@ -91,9 +91,10 @@ module Syntropy
       # @return [void]
       def handle_error(request, err)
         case err
+        when Errno::ECONNRESET
+          # ignore
         when SystemCallError
-          log_error(err, 'I/O error')
-          false
+          log_error(err, 'Syscall error')
         when ProtocolError
           log_error(err, err.message)
           respond(request, err.message, ':status' => err.http_status)
@@ -218,16 +219,16 @@ module Syntropy
       # @param done [boolean] whether the response is completed
       # @return [void]
       def send_chunk(request, chunk, done: false)
-        data = +''
-        data << "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n" if chunk
-        data << EMPTY_CHUNK if done
-        return if data.empty?
-
-        @machine.send(@fd, data, data.bytesize, SEND_FLAGS)
-        return if @done || !done
+        return if @done
 
-        @logger&.info(request: request, response_headers: @response_headers)
-        @done = true
+        len = chunk.is_a?(IO::Buffer) ? chunk.size : chunk.bytesize
+        if !done
+          @io.write(len.to_s(16),"\r\n", chunk, "\r\n")
+        else
+          @io.write(len.to_s(16),"\r\n", chunk, "\r\n0\r\n\r\n")
+          @logger&.info(request: request, response_headers: @response_headers)
+          @done = true
+        end
       end
 
       # Finishes the response to the current request. If no headers were sent,
@@ -257,25 +258,12 @@ module Syntropy
           env[:headers] = cache_headers
         end
 
-        maxlen = env[:max_len] || 65_536
-        buf = String.new(capacity: maxlen)
-        headers_sent = nil
-        loop do
-          res = @machine.read(fd, buf, maxlen, 0)
-          if res < maxlen && !headers_sent
-            return respond(req, buf, env[:headers])
-          elsif res == 0
-            return finish(req)
-          end
-
-          if !headers_sent
-            send_headers(req, env[:headers])
-            headers_sent = true
-          end
-          done = res < maxlen
-          send_chunk(req, buf, done: done)
-          return if done
-        end
+        io = @machine.io(fd, :file)
+        send_headers(req, env[:headers])
+        io.read_each { send_chunk(req, it) }
+        finish(req)
+      ensure
+        @machine.close_async(fd) if fd
       end
 
       # Closes the connection.

data/lib/syntropy/json_api.rb

@@ -25,11 +25,6 @@ module Syntropy
         ':status'       => status,
         'Content-Type'  => 'application/json'
       )
-    rescue => e
-      puts '*' * 40
-      p e
-      p e.backtrace.join
-      puts
     end
 
     private
@@ -39,20 +34,16 @@ module Syntropy
     # @param req [Syntropy::Request]
     def __invoke__(req)
       q = req.validate_param(:q, String).to_sym
-      response = case req.method
+      case req.method
       when 'get'
-        __invoke_get__(q, req)
+        response = __invoke_get__(q, req)
       when 'post'
-        __invoke_post__(q, req)
+        response = __invoke_post__(q, req)
       else
         raise Syntropy::Error.method_not_allowed
       end
       [{ status: 'OK', response: response }, HTTP::OK]
     rescue StandardError => e
-      if !e.is_a?(Syntropy::Error)
-        p e
-        p e.backtrace
-      end
       __error_response__(e)
     end
 

data/lib/syntropy/kv_store.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Syntropy
+  # The KVStore class implements an SQLite-backed key-value store
+  class KVStore
+    attr_reader :table_name
+
+    def initialize(db, table_name)
+      @table_name = table_name
+      setup_queries
+      apply_schema(db)
+    end
+
+    def get(db, key)
+      db.query_single_splat(@sql_get, key)
+    end
+
+    def set(db, key, value)
+      db.execute(@sql_set, key, value)
+    end
+
+    def setex(db, key, value, ttl)
+      db.execute(@sql_setex, key, value, ttl ? Time.now.to_f + ttl : nil)
+    end
+
+    def sweep(db)
+      db.execute(@sql_sweep, Time.now.to_f)
+    end
+
+    private
+
+    def apply_schema(db)
+      db.execute <<~SQL
+        create table if not exists #{@table_name} (key text primary key, value, expires float);
+        create index if not exists idx_#{@table_name}_expires on #{@table_name} (expires) where expires is not null;
+      SQL
+    end
+
+    def setup_queries
+      @sql_get = "select value from #{@table_name} where key = ?"
+
+      @sql_set = <<~SQL
+        insert into #{@table_name} (key, value)
+        values($1, $2)
+        on conflict (key) do update set value = $2, expires = null
+      SQL
+
+      @sql_setex = <<~SQL
+        insert into #{@table_name} (key, value, expires)
+        values($1, $2, $3)
+        on conflict (key) do update set value = $2, expires = $3
+      SQL
+
+      @sql_sweep = "delete from #{@table_name} where expires < ?"
+    end
+  end
+end

data/lib/syntropy/logger.rb

@@ -80,49 +80,40 @@ module Syntropy
     # @param o [Hash] raw entry
     # @return [Hash] log entry
     def make_entry(level, o)
-      if o[:request]
-        make_request_entry(level, o)
-      elsif o[:error]
-        make_error_entry(level, o)
-      else
-        make_hash_entry(level, o)
-      end
+      t = Time.now
+      entry = o.merge(
+        level:        level.to_s,
+        ts:           t.to_i,
+        ts_s:         t.iso8601,
+        message:      o[:message]
+      )
+
+      entry.merge!(error_info(error)) if (error = o.delete(:error))
+      entry.merge!(request_info(request)) if (request = o.delete(:request))
+
+      entry
     end
 
-    # Makes an error log entry.
+    # Extracts error info from given error object
     #
-    # @param level [Symbol] log level
-    # @param o [Hash] input entry
-    # @return [Hash] output entry
-    def make_error_entry(level, o)
-      err = o[:error]
-      t = Time.now
+    # @param error [Exception] exception
+    # @return [Hash] error info
+    def error_info(error)
       {
-        level:  level.to_s,
-        ts:     t.to_i,
-        ts_s:   t.iso8601
-      }.merge(o).merge(
-        error: "#{err.class}: #{err.message}",
-        backtrace: err.backtrace
-      )
+        error: "#{error.class}: #{error.message}",
+        backtrace: error.backtrace
+      }
     end
 
-    # Makes a request log entry.
+    # Extracts request info from given request object
     #
-    # @param level [Symbol] log level
-    # @param o [Hash] input entry
-    # @return [Hash] output entry
-    def make_request_entry(level, o)
-      request = o[:request]
+    # @param request [Syntropy::Request] request
+    # @return [Hash] request info
+    def request_info(request)
       request_headers = request.headers
-      response_headers = o[:response_headers]
+      response_headers = request.response_headers
       elapsed = monotonic_clock - request.start_stamp
-      t = Time.now
       {
-        level:        level.to_s,
-        ts:           t.to_i,
-        ts_s:         t.iso8601,
-        message:      o[:message] || 'HTTP request done',
         client_ip:    request.forwarded_for || '?',
         http_method:  request_headers[':method'].upcase,
         user_agent:   request_headers['user-agent'],
@@ -132,20 +123,6 @@ module Syntropy
       }
     end
 
-    # Makes a request log entry.
-    #
-    # @param level [Symbol] log level
-    # @param o [Hash] input entry
-    # @return [Hash] output entry
-    def make_hash_entry(level, hash)
-      t = Time.now
-      {
-        level:  level.to_s,
-        ts:     t.to_i,
-        ts_s:   t.iso8601
-      }.merge(hash)
-    end
-
     # Returns the monotonic clock.
     #
     # @return [Float]

data/lib/syntropy/markdown.rb

@@ -137,7 +137,7 @@ module Syntropy
       # @return [Array<atts, md>] Array containing attributes and clean markdown
       def parse_md(atts, md)
         md = parse_content(atts, md)
-        [atts, md]
+        [atts, md.chomp]
       end
 
       # Creates a Markdown file contoroller for the given environment,