syntropy 0.40.0 → 0.42.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 96429a90f9e886ffafc25c1e4c88a46157fb79751d21fdbed7af10252ff5d16e
-  data.tar.gz: 040ebeaf507ada9e986847b7630739f9451c88d5384c736fd632f55d4928bff2
+  metadata.gz: 30756c68cd868dbde85d4f7097b2e3196de2dd60cd8954084aef1789e972b50c
+  data.tar.gz: 7e3a60536d78429dacc1f7ff5cdd7fdb5f89d3d7f584f73a60f20c1ac56927e6
 SHA512:
-  metadata.gz: f75c54e095c15c1742786f712062142619a0e7c23ee22295f3dc93825d476027ab26a962054ad3cf868f8d89045af2c3c29733b648d9042ba2594595783ab234
-  data.tar.gz: d60a7f2d439b615db027c9f678ab4fba2aae38deb1c68b9d9e5b29bdbe22d282f7d14e1111458c97e3b7ad004eef1a428714376d40bd1370fa668e08765fe739
+  metadata.gz: 6a5a253f3a9ee2b0253aa44a30381d48daafb1a468e7775af95d3dafa174104ad5cc615f800dde75164f2e83fff798725f5f1ac3c9e518018042076598048f17
+  data.tar.gz: fc7fc0874e17c04145a0df875347c144ced4cc1ecb4e2357813cd471c622546af42dc14acf7d338c3069eb00959ea87f60917d8d0c1fc1856147e0080e200bf2

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+# 0.42.0 2026-10-04
+
+- Add `Syntropy::Collection`
+  - Support for nested collections
+  - Support for invalidation on collection file change
+- Improve static file serving
+- Read modules using `UM#file_read`
+- Refactor log entry computation
+- Refactor and simplify storage layer
+
+# 0.41.0 2026-08-25
+
+- Don't log ECONNRESET errors
+
 # 0.40.0 2026-08-24
 
 - Update UringMachine

data/TODO.md

@@ -1,6 +1,19 @@
-## Immediate
+## Extralite doc site
 
-- [ ] Nicer logging in development mode:
+- Similar to Papercraft site
+- Implement collection with code from papercraft.noteflakes.com
+- Further take ideas from discussion below
+- Implement auto light/dark CSS theme
+- Write docs
+
+## Logging
+
+- Make it possible to use different logger implementations, or maybe chain
+  multiple logger implementations, such that we could emit for example both to
+  STDOUT and to a database.
+- Add optional SQLite-log store (+optional log viewing in Admin interface)
+
+## Nicer logging in development mode:
 
 Nice terminal formatting with colors:
 
@@ -34,6 +47,128 @@ sharon@nf1:~/tmp/capatest2$ npx serve dist
  INFO  Gracefully shutting down. Please wait...
 ```
 
+```
+sharon@nf1 ~/ $ be syntropy serve
+
+   ooo
+  ooooo
+   ooo vvv       Syntropy - a web framework for Ruby
+    o vvvvv      --------------------------------------
+    |  vvv o    https://github.com/digital-fabric/syntropy
+   :|:::|::|:
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+2026-06-07 08:04:23 <== GET /
+2026-06-07 08:04:23 ==> 200 (42ms)
+2026-06-07 08:04:23 <== GET /assets/index-edfizW3i.js
+2026-06-07 08:04:23 <== GET /assets/index-PVjztr5e.css
+2026-06-07 08:04:23 ==> 200 (3ms)
+2026-06-07 08:04:23 ==> 200 (12ms)
+
+```
+
+Verbose output (`-v`):
+
+```
+2026-06-07 08:04:23 <== GET /
+  Host: localhost:3000
+  User-Agent: ...
+  Cookie: ...
+  Accept: ...
+2026-06-07 08:04:23 ==> 200 (42ms)
+  Server: syntropy
+  Transfer-Encoding: chunked
+  ...
+2026-06-07 08:04:23 <== GET /assets/index-edfizW3i.js
+  ...
+2026-06-07 08:04:23 <== GET /assets/index-PVjztr5e.css
+  ...
+2026-06-07 08:04:23 ==> 200 (3ms)
+  ...
+2026-06-07 08:04:23 ==> 200 (12ms)
+  ...
+```
+
+## Background jobs
+
+- Backed by SQLite db
+- Recurring jobs
+- Scheduled jobs
+- Retries
+- CLI command `syntropy background`
+- Run automatically when running `syntropy serve` in dev mode
+
+## Admin Dashboard
+
+- Server state / stats
+- Background jobs state / stats
+- For resources:
+
+  ```ruby
+  # /app/admin+.rb
+  export admin_ui(
+    resources: {
+      posts: {
+        type: 'crud',
+        module: '/storage/posts',
+        primary_key: :id,
+        editor_fields: {
+          id: hidden
+          title: text,
+          body: textarea
+        },
+        actions: {
+          list:  ->(s) { s.list() },
+          list: 
+        }
+      }
+    }
+  )
+  ```
+
+  Or, maybe:
+
+  ```ruby
+  # /app/admin/index.rb
+  Admin = import '/admin/.controller'
+  export Admin.homepage
+
+
+  # /app/admin/.controller.rb
+  require 'syntropy/admin'
+  export Syntropy::Admin.new(
+    # config
+  )
+
+
+  # /app/admin/resources/posts.rb
+  Admin = import '/admin/.controller'
+  Posts = import '/storage/posts'
+
+  Resource = Admin.resource(
+    name: 'Posts',
+    fields: {
+      id: hidden
+      title: text,
+      body: textarea
+    }
+  )
+  export Resource.controller
+
+  def list(req)
+    list = Posts.list
+  end
+  
+    actions: {
+      list:  ->(s) { s.list() },
+      list: 
+    }
+  )
+  ```
+
+
+# Pub/sub
+
 - [ ] Ability to load modules from builtin applet
   - [ ] Can we mount them on the app's module loader?
 
@@ -41,46 +176,56 @@ sharon@nf1:~/tmp/capatest2$ npx serve dist
 
 - [ ] Pub/sub
   - [ ] Ruby side
+
+  ```ruby
+  @firehose = @app.import('/.syntropy/firehose')
+
+  def on_change(v)
+    @firehose.publish({ kind: 'value_changed', value: v })
+  end
+
+  ## in a pub/sub controller
+  @firehose = @app.import('/.syntropy/firehose')
+
+  def call(req)
+    @firehose.stream_sse(req)
+  end
+  ```
+
   - [ ] JS side
+
+  ```javascript
+  await window.syntropy.firehose.process(async (evt, next) => {
+    if (evt.kind == 'value_changed')
+      await valueChanged(evt.value)
+    else if (next)
+      await next(evt);
+  });
+  ```
+
   - [ ] Reimplement `auto_refresh` using a *default* event bus provided by
         Syntropy
 
 - [ ] An alternative to the pub/sub design - add streaming responses to the
   JSON/jS API (using SSE).
 
-## Collections
-
-- [ ] Collection - treat directories and files as collections of data.
+## Scaffolding
 
-  Kind of similar to the routing tree, but instead of routes it just takes a
-  bunch of files and turns it into a dataset. Each directory is a "table" and is
-  composed of zero or more files that form rows in the table. Supported file
-  formats:
+In rails, you can generate specific parts of the app, like a model or a
+controller, or a whole set of model, view, controller code for a given entity
+(scaffolding.)
 
-  - foo.md - markdown with optional front matter
-  - foo.json - JSON record
-  - foo.yml - YAML record
+Does this make sense for Syntropy? The different kinds of code generated:
 
-  API:
+- a model with CRUD methods for the given entity.
+- a set of CRUD controllers (spread over multiple files).
+- a set of CRUD views corresponding to the controller.
+- a set of tests for model, controller, view.
 
-  ```ruby
-  Articles = @app.collection('_articles/*.md')
-  article = Articles.last_by(&:date)
-
-  article.title #=>
-  article.date #=>
-  article.layout #=>
-  article.render_proc #=> (load layout, apply article)
-  article.render #=> (render to HTML)
-
-  # there should also be methods for creating, updating and deleting of
-  # articles/items.
-  ...
-  ```
-
-- [ ] Improve serving of static files:
-  - [ ] support for compression
-  - [ ] add `Request#render_static_file(route, fn)
+But in fact they don't make much sense separately, except maybe for the model.
+So we follow Rail's example and implement a code generator. This is especially
+useful for working with controllers. Since the controller code is spread over
+four files, this is not something trivial.
 
 ## Missing for a first public release
 
@@ -153,3 +298,161 @@ end
   2.5. If docker compose services are running, restart
   2.6. Otherwise, start services
   2.7. Verify service is running correctly
+
+## Syntropy Flow - Live Templates
+
+The idea is to have something that works like LiveView, but without WebSocket.
+Instead, we'll use SSE in conjunction with a controller URL that provides both
+updates (through SSE), and receives user interaction from the browser with POST
+requests.
+
+The compiler needs to be able to compile:
+
+- The HTML
+- The initial update tree (statics + dynamics)
+- Updated dynamic values
+
+Supposing that the HTML and the initial update tree are rendered at once. Let's
+say we have the following template:
+
+```ruby
+->(name:) {
+  html {
+    head { title "My live page" }
+    body {
+      p "Hello, #{name}!"
+    }
+  }
+}
+```
+
+Which normally compiles to:
+
+```ruby
+->(__buffer__, name:) {
+  __buffer__
+    .<<("<!DOCTYPE html><html><head><title>My live page</title></head><body><p>")
+    .<<(ERB::Escape.html_escape(("Hello, #{name}!")))
+    .<<("</p></body></html>")
+  __buffer__
+}
+```
+
+A live template would look something like:
+
+```ruby
+# __flow_url__ is the URL used for the SSE connection, it includes an id for the
+# specific connection.
+initial = ->(__flow_url__, name:) {
+  __html__ = +''
+  __static__ = []; __dynamic__ = {}; __dynamic_counter__ = 0;
+  
+  __html__
+    .<<(
+      s = "<!DOCTYPE html><html><head><title>My live page</title></head><body><p>"
+      __static__ << s
+      s
+    )
+    .<<(
+      s = ERB::Escape.html_escape(("Hello, #{name}!"))
+      __dynamic__[__dynamic_counter__] = s
+      __dynamic_counter__ += 1
+      s
+    )
+    .<<(
+      s = "</p></body><script src=\"#{__flow_url__}\"></script></html>"
+      __static__ << s
+      s
+    )
+  {
+    html: __html__,
+    update_tree = {
+      s: __static__,
+      d: __dynamic__
+    }
+  }
+}
+```
+
+The controller would look something like:
+
+```ruby
+@template = import '/views/hello_world'
+
+export flow_controller do
+  def setup
+    assigns[:name] = 'world'
+    assigns[:counter] ||= 1
+    @updater = @machine.spin {
+      @machine.periodically(3) {
+        assigns[:name] = "world (#{assignes[:counter] += 1})"
+      }
+    }
+  end
+
+  def teardown
+    @machine.cancel(@updater)
+  end
+end
+```
+
+The flow_controller can handle four kinds of requests:
+
+- `GET /hello` - This is the initial render, where it will:
+  - generate a unique flow id which will be embedded in a flow url (see below)
+  - render `initial` compiled template, and store the update tree in a short
+    term hash (which is evacuated periodically for failed connections)
+  - respond with the HTML
+- `GET /hello?flow=setup` - The JS payload to start the SSE connection
+  - boilerplate code for setting up the SSE connection
+  - include inline the initial update tree containing static and dynamic parts
+- `GET /hello?sse&fid=xxxx` - the SSE long-running connection, on which
+  updates are sent. The flow id identifies the flow session, which permits
+  reconnection in case of comm error. On connection, the controller will emit an
+  initial update with the update tree (see below).
+
+- `POST /hello?fid=xxxx` - for responding to user interaction.
+
+Whenever a value in `assigns` changes, the *update* template is rerendered.
+Here's how the compiled update template looks:
+
+```ruby
+initial = ->(name:) {
+  __dynamic__ = {}; __dynamic_counter__ = 0;
+  
+  s = ERB::Escape.html_escape(("Hello, #{name}!"))
+  __dynamic__[__dynamic_counter__] = s
+  __dynamic_counter__ += 1
+
+  __dynamic__
+}
+```
+
+So, the initial update for this template will look something like the following:
+
+```json
+{
+  s: [
+    "<!DOCTYPE html><html><head><title>My live page</title></head><body><p>",
+    "</p></body><script src=\"/hello?sse&fid=12345678\"></script></html>"
+  ],
+  d: {
+    0: "world"
+  }
+}
+```
+
+Subsequent updates will look like:
+
+```json
+{
+  d: {
+    0: "world (10:27:32)"
+  }
+}
+```
+
+On the client side, each time an update is received on the SSE connection, it
+includes the dynamic values. Those are diffed against the previous values, the
+entire page HTML is zipped from the static and dynamic parts, and then morphdom
+is used to patch the DOM with the relevant changes.

data/cmd/new/template/app/_lib/storage.rb

@@ -1,11 +1,11 @@
 export self
 
 def connection_pool
-  @connection_pool ||= Storage::ConnectionPool.new(@machine, @env[:config][:storage][:path], 4)
+  @connection_pool ||= ConnectionPool.new(@machine, @env[:config][:storage][:path], 4)
 end
 
 def schema
-  Storage::Schema.new(module_loader: @module_loader, schema_root: '_schema')
+  Schema.new(module_loader: @module_loader, schema_root: '_schema')
 end
 
 def migrate!

data/cmd/new/template/config/test.rb

@@ -1,5 +1,5 @@
 export({
   storage: {
-    path: ENV['DATABASE_PATH'] || Syntropy.tmp_path('test-db')
+    path: ENV['DATABASE_PATH'] || tmp_path('test-db')
   }
 })

data/examples/agent/config/test.rb

@@ -1,5 +1,5 @@
 export({
   storage: {
-    path: ENV['DATABASE_PATH'] || Syntropy.tmp_path('test-db')
+    path: ENV['DATABASE_PATH'] || tmp_path('test-db')
   }
 })

data/examples/blog/app/_lib/posts.rb

@@ -1,9 +1,13 @@
 Storage = import '/_lib/storage'
 
-class PostStore < Syntropy::Storage::Store
+class PostStore
+  def initialize(cp)
+    @cp = cp
+  end
+  
   # @return [Integer] post id
   def create(title, body)
-    query_single_value <<~SQL, title:, body:
+    @cp.query_single_value <<~SQL, title:, body:
       insert into posts (title, body)
       values (:title, :body)
       returning id;
@@ -12,7 +16,7 @@ class PostStore < Syntropy::Storage::Store
 
   # @return [void]
   def update(id, title, body)
-    execute <<~SQL, id:, title:, body:
+    @cp.execute <<~SQL, id:, title:, body:
       update posts
       set title = :title, body = :body
       where id = :id
@@ -21,7 +25,7 @@ class PostStore < Syntropy::Storage::Store
 
   # @return [void]
   def delete(id)
-    execute <<~SQL, id:
+    @cp.execute <<~SQL, id:
       delete from posts
       where id = :id
     SQL
@@ -29,7 +33,7 @@ class PostStore < Syntropy::Storage::Store
 
   # return [Hash]
   def get(id)
-    query_single_row <<~SQL, id:
+    @cp.query_single_row <<~SQL, id:
       select id, title, body
       from posts
       where id = :id
@@ -38,7 +42,7 @@ class PostStore < Syntropy::Storage::Store
 
   # return [Array<Hash>]
   def get_all
-    query <<~SQL
+    @cp.query <<~SQL
       select id, title, body
       from posts
       order by id desc

data/examples/blog/app/_lib/storage.rb

@@ -1,11 +1,11 @@
 export self
 
 def connection_pool
-  @connection_pool ||= Storage::ConnectionPool.new(@machine, @env[:config][:storage][:path], 4)
+  @connection_pool ||= ConnectionPool.new(@machine, @env[:config][:storage][:path], 4)
 end
 
 def schema
-  Storage::Schema.new(module_loader: @module_loader, schema_root: '_schema')
+  Schema.new(module_loader: @module_loader, schema_root: '_schema')
 end
 
 def migrate!

data/examples/blog/app/posts/[id]/index.rb

@@ -23,7 +23,7 @@ def post(req)
   raise BadRequestError, "Failed to update post" if updated != 1
 
   req.flash[:notice] = 'Post was successfully updated.'
-  req.redirect "/posts/#{id}", Syntropy::HTTP::SEE_OTHER
+  req.redirect "/posts/#{id}", HTTP::SEE_OTHER
 end
 
 def delete(req)
@@ -33,7 +33,7 @@ def delete(req)
   raise BadRequestError, 'Failed to delete post' if deleted != 1
 
   req.flash[:notice] = 'Post was successfully destroyed.'
-  req.redirect '..', Syntropy::HTTP::SEE_OTHER
+  req.redirect '..', HTTP::SEE_OTHER
 end
 
 @template = @layout.apply { |post:, req:, **props|

data/examples/blog/config/test.rb

@@ -1,5 +1,5 @@
 export({
   storage: {
-    path: ENV['DATABASE_PATH'] || Syntropy.tmp_path('test-db')
+    path: ENV['DATABASE_PATH'] || tmp_path('test-db')
   }
 })