syntropy 0.40.0 → 0.42.0

data/lib/syntropy/module_loader.rb

@@ -38,6 +38,7 @@ module Syntropy
       @loading = Set.new
       @lock = UM::Mutex.new
       @lock_holder = nil
+      @invalidation_patterns = Hash.new { |h, k| h[k] = [] }
     end
 
     # Loads a module (if not already loaded) and returns its export value.
@@ -80,12 +81,27 @@ module Syntropy
         ref = @fn_map[fn]
         invalidate_ref(ref) if ref
         invalidate_collection_modules
+        invalidate_by_pattern(fn)
       end
     end
 
+    # Adds an invalidation pattern. When a ref has been invalidated (using
+    # #invalidate_fn), the loader will check for matches against any
+    # invalidation patterns, and will invalidate any corresponding refs. This
+    # mechanism allows hot reload for collections.
+    #
+    # @param ref [String] module ref
+    # @param pattern [String] pattern normalized to app root
+    # @return [void]
+    def invalidate_on_file_change(ref, pattern)
+      # pattern should be absolute in order to match against fn
+      pattern = File.join(@app_root, pattern)
+      @invalidation_patterns[pattern] << ref
+    end
+
     private
 
-    # Synchronizes access to the module loader state
+    # Synchronizes access to the module loader state.
     #
     # @return [any] block return value
     def lock
@@ -107,6 +123,11 @@ module Syntropy
     # @param ref [String] module reference
     # @return [void]
     def invalidate_ref(ref)
+      if ref == '/'
+        invalidate_ref('/index')
+        invalidate_ref('/index+')
+      end
+
       entry = @modules.delete(ref)
       return if !entry
 
@@ -125,6 +146,19 @@ module Syntropy
       refs.each { invalidate_ref(it) }
     end
 
+    # Iterates through invalidation patterns and for each match invalidates the
+    # corresponding refs.
+    #
+    # @param fn [String] filename
+    # @return [void]
+    def invalidate_by_pattern(fn)
+      invalidated = Set.new
+      @invalidation_patterns.each do |pat, refs|
+        refs.each { invalidated << it } if File.fnmatch(pat, fn)
+      end
+      invalidated.each { invalidate_ref(it) }
+    end
+
     # Registers reverse dependencies for the given module reference.
     #
     # @param ref [String] module reference
@@ -165,7 +199,7 @@ module Syntropy
     def do_load_module(ref, fn, raise_on_missing_export:)
       @loading << ref
       @fn_map[fn] = ref
-      code = read_file(fn)
+      code = @machine.file_read(fn)
       env = @env.merge(module_loader: self, ref: clean_ref(ref))
       mod = Syntropy::ModuleContext.new(env, code, fn, @extensions)
       add_dependencies(ref, mod.__dependencies__)
@@ -187,18 +221,6 @@ module Syntropy
       @loading.delete(ref)
     end
 
-    # Returns a file's content.
-    #
-    # @param fn [String] filename
-    # @return [String] file content
-    def read_file(fn)
-      @machine.open(fn, UM::O_RDONLY) { |fd|
-        buf = +''
-        res = @machine.read(fd, buf, 1 << 20)
-        buf
-      }
-    end
-
     # Cleans up a module reference specifier, turning /index into /
     #
     # @param ref [String] input ref
@@ -270,6 +292,11 @@ module Syntropy
       @module_loader = env[:module_loader]
       @app = env[:app]
       @ref = env[:ref]
+
+      mp = env[:mount_path]
+      @url = mp && (@ref == '/' ? mp : File.join(mp, @ref))
+      @fn  = fn
+
       @logger = env[:logger]
       @__dependencies__ = []
       singleton_class.const_set(:MODULE, self)
@@ -320,6 +347,18 @@ module Syntropy
       self
     end
 
+    # Causes the module to be invalidated (reloaded) when a change is detected
+    # in any file that matches the given pattern. The pattern is relative to the
+    # current module's ref. The pattern normally includes a wildcard:
+    #
+    #     invalidate_on_file_change('_articles/*')
+    #
+    # @param pattern [String] pattern in relative path
+    # @return [void]
+    def invalidate_on_file_change(pattern)
+      @module_loader.invalidate_on_file_change(@ref, normalize_import_ref(pattern))
+    end
+
     # Normalize an import reference, turning a relative path into an absolute one.
     #
     # @param ref [String] input ref

data/lib/syntropy/schema.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Syntropy
+  class Schema
+    def initialize(module_loader: nil, schema_root: '_schema', &)
+      @migrations = {}
+      @module_loader = module_loader
+      @schema_root = schema_root
+      load_schema_from_modules if @module_loader
+      run_schema_dsl(&) if block_given?
+    end
+
+    def apply(connection_pool)
+      execute_migrations(connection_pool)
+    end
+
+    def current_version(connection_pool)
+      connection_pool.with_db do |db|
+        get_schema_version(db)
+      end
+    end
+
+    private
+
+    def load_schema_from_modules
+      modules = @module_loader.list(@schema_root)
+      modules.each do |name|
+        @migrations[File.basename(name)] = @module_loader.load(name)
+      end
+    end
+
+    class SchemaDSLRunner
+      def initialize(migrations)
+        @migrations = migrations
+      end
+
+      def run(&)
+        instance_eval(&)
+      end
+
+      private
+
+      def initial(&block)
+        @migrations['0000'] = block
+      end
+
+      def version(key, &block)
+        @migrations[key] = block
+      end
+    end
+
+    def run_schema_dsl(&)
+      SchemaDSLRunner.new(@migrations).run(&)
+    end
+
+    def execute_migrations(connection_pool)
+      connection_pool.with_db do |db|
+        current_version = get_schema_version(db)
+        migrations_keys = @migrations.keys.sort
+        migrations_keys.select! { it > current_version } if current_version
+
+        migrations_keys.each do |key|
+          db.transaction do
+            @migrations[key].(db)
+            set_schema_version(db, key)
+            current_version = key
+          end
+        end
+
+        current_version
+      end
+    end
+
+    def get_schema_version(db)
+      db.execute <<~SQL
+        create table if not exists __syntropy_schema__(
+          k text primary key,
+          v text
+        );
+      SQL
+      db.query_single_splat <<~SQL
+        select v from __syntropy_schema__
+        where k = 'version'
+      SQL
+    end
+
+    def set_schema_version(db, version)
+      db.execute <<~SQL, version
+        insert into __syntropy_schema__ (k, v)
+        values ('version', $1)
+        on conflict(k) do update set v = $1
+      SQL
+    end
+  end
+end

data/lib/syntropy/storage.rb

@@ -1,6 +1,44 @@
 # frozen_string_literal: true
 
-require 'syntropy/storage/connection_pool'
-require 'syntropy/storage/schema'
-require 'syntropy/storage/store'
-require 'syntropy/storage/prepared_query'
+require 'extralite'
+require 'syntropy/connection_pool'
+require 'syntropy/schema'
+require 'syntropy/kv_store'
+
+module Syntropy
+  class Storage
+    attr_reader :connection_pool, :schema
+    
+    def initialize(machine, module_loader, config)
+      @machine = machine
+      @module_loader = module_loader
+      @config = config
+
+      raise Syntropy::Error, 'Missing storage config' if !config
+      raise Syntropy::Error, 'Missing storage config' if !config[:path]
+
+      @connection_pool ||= ConnectionPool.new(
+        @machine,
+        config[:path],
+        config[:concurrency] || 4
+      )
+
+      @schema = Schema.new(
+        module_loader: @module_loader,
+        schema_root: @config[:schema_root] || '_schema'
+      )
+    end
+
+    def query(*, **, &)
+      connection_pool.with_db { it.query(*, **, &) }
+    end
+
+    def query_single_value(*, **, &)
+      connection_pool.with_db { it.query_splat(*, **, &) }
+    end
+
+    def execute(*, **)
+      connection_pool.with_db { it.execute(*, **) }
+    end
+  end
+end

data/lib/syntropy/test.rb

@@ -235,7 +235,7 @@ module Syntropy
     # @return [any] parsed JSON object
     def response_json
       raise if response_content_type != 'application/json'
-      JSON.parse(response_body)
+      JSON.parse(response_body, symbolize_names: true)
     end
 
     # Returns the response content MIME type.

data/lib/syntropy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Syntropy
-  VERSION = '0.40.0'
+  VERSION = '0.42.0'
 end

data/lib/syntropy.rb

@@ -17,6 +17,7 @@ require 'syntropy/papercraft_extensions'
 require 'syntropy/routing_tree'
 require 'syntropy/json_api'
 require 'syntropy/side_run'
+require 'syntropy/collection'
 require 'syntropy/version'
 
 # Syntropy is a web framework for building web apps in Ruby. Syntropy uses
@@ -86,7 +87,7 @@ module Syntropy
         logger: nil,
         machine: UM.new(size: 8)
       )
-      loader = ModuleLoader.new(loader_env)
+      loader = ModuleLoader.new(loader_env, extensions: ControllerExtensions)
       if (config = loader.load(env[:mode], raise_on_missing_export: false))
         env[:config] = config
       end

data/syntropy.gemspec

@@ -21,9 +21,9 @@ Gem::Specification.new do |s|
   s.required_ruby_version = '>= 3.4'
   s.executables = ['syntropy']
 
-  s.add_dependency 'extralite',     '~>3.0.1'
+  s.add_dependency 'extralite',     '~>3.1.3'
   s.add_dependency 'papercraft',    '~>3.2.2'
-  s.add_dependency 'uringmachine',  '~>1.0.4'
+  s.add_dependency 'uringmachine',  '~>1.0.6'
 
   s.add_dependency 'json'
   s.add_dependency 'base64'

data/test/fixtures/app/_lib/dep_invalidate.rb

@@ -0,0 +1,5 @@
+Foo = import '/_lib/self'
+
+invalidate_on_file_change('circular/**')
+
+export self

data/test/fixtures/collection/basic/_articles/2026-06-30-foo.md

@@ -0,0 +1,6 @@
+---
+title: FooFoo
+category: [a, b]
+author: Bar Baz
+---
+Foo foo foo.

data/test/fixtures/collection/basic/_articles/2026-07-13-bar.md

@@ -0,0 +1,6 @@
+---
+title: BarBar
+category: [b, c]
+author: Baz Foo
+---
+Bar bar bar.

data/test/fixtures/collection/basic/_articles/2026-08-07-baz.md

@@ -0,0 +1,6 @@
+---
+title: BazBaz
+category: [c, a]
+author: Foo Bar
+---
+Foo foo foo.

data/test/fixtures/collection/basic/index+.rb

@@ -0,0 +1,23 @@
+begin
+  @articles = collection('_articles', url_base: @url)
+rescue => e
+  p e
+  p e.backtrace
+  exit!
+end
+
+export ->(req) do
+  if req.path == @url
+    list = @articles.list.map { it[:ref] }
+    req.respond_json(list)
+  elsif (article = @articles.get(req.path))
+    o = article.slice(:fn, :ref, :url, :type, :attributes, :body)
+    req.respond_json(o)
+  else
+    raise Error.not_found if !article
+  end
+rescue => e
+  p e
+  p e.backtrace
+  exit!
+end

data/test/fixtures/collection/docs/_pages/01-foo/01-foo.md

@@ -0,0 +1,4 @@
+---
+title: FooFoo Foo
+---
+Foo foo foo.

data/test/fixtures/collection/docs/_pages/01-foo/02-bar.md

@@ -0,0 +1,4 @@
+---
+title: FooFoo Bar
+---
+Bar bar bar.

data/test/fixtures/collection/docs/_pages/01-foo/_index.yml

@@ -0,0 +1 @@
+title: "FooFoo"

data/test/fixtures/collection/docs/_pages/02-bar/01-bar.md

@@ -0,0 +1,4 @@
+---
+title: BarBar Bar
+---
+Bar bar bar.

data/test/fixtures/collection/docs/_pages/02-bar/02-baz.md

@@ -0,0 +1,4 @@
+---
+title: BarBar Baz
+---
+Baz baz baz.

data/test/fixtures/collection/docs/_pages/02-bar/_index.yml

@@ -0,0 +1 @@
+title: "BarBar"

data/test/fixtures/collection/docs/index+.rb

@@ -0,0 +1,16 @@
+@pages = collection('_pages', url_base: @url)
+
+export ->(req) do
+  if req.path == @url
+    list = @articles.list.map { it[:ref] }
+    req.respond_json(list)
+  elsif (page = @pages.get(req.path))
+    if page[:type] == :directory
+      req.respond_json(page.slice(:type, :title))
+    else
+      req.respond_json(page.slice(:type, :title, :body))
+    end
+  else
+    raise Error.not_found if !article
+  end
+end

data/test/test_app.rb

@@ -56,28 +56,28 @@ class AppTest < Minitest::Test
     assert_equal HTTP::NOT_FOUND, req.response_status
 
     req = @test_harness.request(':method' => 'GET', ':path' => '/test/api?q=get')
-    assert_equal({ 'status' => 'OK', 'response' => 0 }, req.response_json)
+    assert_equal({ status: 'OK', response: 0 }, req.response_json)
 
     req = @test_harness.request(':method' => 'POST', ':path' => '/test/api?q=get')
     assert_equal HTTP::METHOD_NOT_ALLOWED, req.response_status
-    assert_equal({ 'status' => 'Error', 'message' => 'Method not allowed' }, req.response_json)
+    assert_equal({ status: 'Error', message: 'Method not allowed' }, req.response_json)
 
     req = @test_harness.request(':method' => 'GET', ':path' => '/test/api/foo?q=get')
-    assert_equal({ 'status' => 'OK', 'response' => 0 }, req.response_json)
+    assert_equal({ status: 'OK', response: 0 }, req.response_json)
 
     req = @test_harness.request(':method' => 'POST', ':path' => '/test/api?q=incr')
-    assert_equal({ 'status' => 'OK', 'response' => 1 }, req.response_json)
+    assert_equal({ status: 'OK', response: 1 }, req.response_json)
 
     req = @test_harness.request(':method' => 'GET', ':path' => '/test/api?q=incr')
     assert_equal HTTP::METHOD_NOT_ALLOWED, req.response_status
-    assert_equal({ 'status' => 'Error', 'message' => 'Method not allowed' }, req.response_json)
+    assert_equal({ status: 'Error', message: 'Method not allowed' }, req.response_json)
 
     req = @test_harness.request(':method' => 'POST', ':path' => '/test/api/foo?q=incr')
-    assert_equal({ 'status' => 'Error', 'message' => 'Teapot' }, req.response_json)
+    assert_equal({ status: 'Error', message: 'Teapot' }, req.response_json)
     assert_equal HTTP::TEAPOT, req.response_status
 
     req = @test_harness.request(':method' => 'POST', ':path' => '/test/api/foo/bar?q=incr')
-    assert_equal({ 'status' => 'Error', 'message' => 'Teapot' }, req.response_json)
+    assert_equal({ status: 'Error', message: 'Teapot' }, req.response_json)
     assert_equal HTTP::TEAPOT, req.response_status
 
     req = @test_harness.request(':method' => 'GET', ':path' => '/test/bar')

data/test/test_cmd_new.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative 'helper'
+require 'syntropy/test'
+
+class CmdNewTemplateTest < Syntropy::Test
+  def env
+    {
+      mode: 'test',
+      app_root: File.join(__dir__, '../cmd/new/template/app'),
+      config_root: File.join(__dir__, '../cmd/new/template/config'),
+      mount_path: '/'
+    }
+  end
+
+  def test_cmd_new_template_server
+    req = get('/foo')
+    assert_equal HTTP::NOT_FOUND, req.response_status
+
+    req = get('/')
+    assert_equal HTTP::OK, req.response_status
+    assert_equal 'text/html', req.response_content_type
+  end
+
+  def test_cmd_new_template_storage
+    storage = load_module('_lib/storage')
+    refute_nil storage
+
+    cp = storage.connection_pool
+    assert_kind_of Syntropy::ConnectionPool, cp
+
+    schema = storage.schema
+    assert_kind_of Syntropy::Schema, schema
+
+    # migration already ran
+    refute_nil schema.current_version(cp)
+  end
+end

data/test/test_collection.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+require_relative 'helper'
+
+class CollectionTest < Minitest::Test
+  COLLECTION_ROOT = File.join(__dir__, 'fixtures/collection/basic/_articles')
+
+  def setup
+    @machine = UM.new
+    @collection = Syntropy::Collection.new(
+      machine:  @machine,
+      root:     COLLECTION_ROOT,
+      url_base: '/apps/blog'
+    )
+  end
+
+  def test_collection_setup
+    assert_equal COLLECTION_ROOT, @collection.root_dir
+    assert_equal '/apps/blog', @collection.url_base
+  end
+
+  def test_collection_list
+    list = @collection.list
+    assert_kind_of Array, list
+    assert_equal 3, list.size
+
+    foo = list[0]
+    assert_kind_of Hash, foo
+    assert_equal({
+      fn:   File.join(COLLECTION_ROOT, '2026-06-30-foo.md'),
+      ref:  '2026-06-30-foo',
+      url:  '/apps/blog/2026-06-30-foo',
+      type: :markdown,
+      attributes: {
+        date:     Date.parse('2026-06-30'),
+        title:    'FooFoo',
+        category: ['a', 'b'],
+        author:   'Bar Baz',
+      },
+      body: 'Foo foo foo.'
+    }, foo.slice(:fn, :ref, :url, :type, :attributes, :body))
+
+    bar = list[1]
+    assert_kind_of Hash, foo
+    assert_equal({
+      fn:   File.join(COLLECTION_ROOT, '2026-07-13-bar.md'),
+      ref:  '2026-07-13-bar',
+      url:  '/apps/blog/2026-07-13-bar',
+      type: :markdown,
+      attributes: {
+        date:     Date.parse('2026-07-13'),
+        title:    'BarBar',
+        category: ['b', 'c'],
+        author:   'Baz Foo',
+      },
+      body: 'Bar bar bar.'
+    }, bar.slice(:fn, :ref, :url, :type, :attributes, :body))
+  end
+
+  def test_collection_get
+    assert_nil @collection.get('/')
+    assert_nil @collection.get('/apps/blog')
+
+    item = @collection.get('/apps/blog/2026-06-30-foo')
+    assert_kind_of Hash, item
+    assert_equal({
+      fn:   File.join(COLLECTION_ROOT, '2026-06-30-foo.md'),
+      ref:  '2026-06-30-foo',
+      url:  '/apps/blog/2026-06-30-foo',
+      type: :markdown,
+      attributes: {
+        date:     Date.parse('2026-06-30'),
+        title:    'FooFoo',
+        category: ['a', 'b'],
+        author:   'Bar Baz',
+      },
+      body: 'Foo foo foo.'
+    }, item.slice(:fn, :ref, :url, :type, :attributes, :body))
+  end
+end
+
+class CollectionAppTest < Syntropy::Test
+  self.env = {
+    app_root: File.join(__dir__, 'fixtures/collection/basic'),
+    mount_path: '/blog'
+  }
+
+  def test_collection_app_controller
+    req = get('/blog/2026-06-30-foo')
+    assert_equal HTTP::OK, req.response_status
+
+    assert_equal ['/index+'], app.module_loader.modules.keys
+
+    app.module_loader.invalidate_fn(File.join(env[:app_root], '_articles/2026-06-30-foo.md'))
+    assert_equal [], app.module_loader.modules.keys
+  end
+end
+
+class NestedCollectionTest < Syntropy::Test
+  self.env = {
+    app_root: File.join(__dir__, 'fixtures/collection/docs'),
+    mount_path: '/docs'
+  }
+
+  def setup
+    super
+    @collection = Syntropy::Collection.new(
+      machine:  @machine,
+      root:     File.join(env[:app_root], '_pages'),
+      url_base: '/docs'
+    )
+  end
+
+  def test_nested_collection_list
+    list = @collection.list
+    assert_equal 4, list.size
+
+    assert_equal 'FooFoo Foo', list[0][:attributes][:title]
+    assert_equal 'FooFoo Bar', list[1][:attributes][:title]
+    assert_equal 'BarBar Bar', list[2][:attributes][:title]
+    assert_equal 'BarBar Baz', list[3][:attributes][:title]
+
+    assert_nil            list[0][:prev]
+    assert_equal list[1], list[0][:next]
+
+    assert_equal list[0], list[1][:prev]
+    assert_equal list[2], list[1][:next]
+
+    assert_equal list[1], list[2][:prev]
+    assert_equal list[3], list[2][:next]
+    
+    assert_equal list[2], list[3][:prev]
+    assert_nil            list[3][:next]
+  end
+
+  def test_nested_collection_dirs
+    root_dir = File.join(env[:app_root], '_pages')
+    root = @collection.root
+    assert_kind_of Hash, root
+
+    assert_equal({
+      fn:   nil,
+      ref:  '',
+      url:  '/docs',
+      type: :directory,
+    }, root.slice(:fn, :ref, :url, :type))
+
+    assert_equal 2, root[:items].size
+
+    assert_equal({
+      fn:   File.join(root_dir, '01-foo/_index.yml'),
+      ref:  '01-foo',
+      url:  '/docs/01-foo',
+      type: :directory
+    }, root[:items][0].slice(:fn, :ref, :url, :type))
+
+    assert_equal({
+      fn:   File.join(root_dir, '02-bar/_index.yml'),
+      ref:  '02-bar',
+      url:  '/docs/02-bar',
+      type: :directory
+    }, root[:items][1].slice(:fn, :ref, :url, :type))
+  end
+end