tep 0.11.4 → 0.11.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 440ba4bc444318eb021fa61653935297e7fee6876f7ce09e06bd45e091f5a14a
-  data.tar.gz: 3babc819f652798441f6f9ecc0f088afb398dae0e977186e9bb3c3e9fc455562
+  metadata.gz: 33b663dcc38571d71f8a729f7ed1f8da6ea8a89e1eb6bd49619beac977da2727
+  data.tar.gz: af7b47e7e3440372325744d1274e70e41216db757ac224ca2364c1a422b5064a
 SHA512:
-  metadata.gz: 514652e89448eef94073b1680228ccbc876f53df63712f580f1c3c8e8b50b0852d0630f456ca26df93844456d7b124ee1681c794e997d41dffd8299e8cb3058d
-  data.tar.gz: be8785d792ad05e0dcde98a04aba8426b967230a42797e47d1536ad67057a7f78bf03f23ded9194c0abd10d3e33c8d0b6023964e7735020d2ca002fb0afc90d1
+  metadata.gz: ed5ce816c591c5464d624e3094798d3ce8c8c03c7128e3881541cf874a0735e62f468d7d4f781e793e6ae02f2bcb642119fdc49157394acb69cb027a014d9b6e
+  data.tar.gz: 484a1cdae65a65cbe5da98d8c2aa22add1b4a562e35ab7cfb38f9137856bcad77ad773b07d191fb425a5e6307fe18a71856d47ad3766e8a03191622314ba2989

data/Makefile

@@ -34,6 +34,16 @@ TEP_PG_LIBS   ?= $(shell \
     (pg_config --libdir 2>/dev/null | sed -e 's|^|-L|' ; echo "-lpq") | tr '\n' ' ')
 export TEP_PG_CFLAGS TEP_PG_LIBS
 
+# OpenSSL cflags / libs for sphttp.c's outbound-TLS include (#148). On Linux
+# the headers/libs are on the default search path so this is usually empty;
+# on macOS Homebrew keeps openssl@3 keg-only (/opt/homebrew/opt/openssl@3),
+# off the default path, so pkg-config supplies the -I (compile) and -L/-lssl
+# /-lcrypto (link). If pkg-config can't find openssl, fall back to the bare
+# libs and let the include path come from the system default.
+TEP_SPHTTP_CFLAGS ?= $(shell pkg-config --cflags openssl 2>/dev/null)
+TEP_SPHTTP_LIBS   ?= $(shell pkg-config --libs openssl 2>/dev/null || echo "-lssl -lcrypto")
+export TEP_SPHTTP_CFLAGS TEP_SPHTTP_LIBS
+
 .PHONY: all clean helper hello sinatra_style bench bench-tep bench-sinatra demo test test-parallel spinel-fresh test-pg vendor-examples vendor-spinelkit doctor
 
 # Re-sync the vendored SpinelKit lib (lib/spinel_kit/, sig/spinel_kit/) from the
@@ -73,7 +83,7 @@ all: spinel-fresh helper hello sinatra_style bench
 helper: spinel-fresh $(LIB_DIR)/sphttp.o $(LIB_DIR)/tep_sqlite.o $(LIB_DIR)/tep_pg.o
 
 $(LIB_DIR)/sphttp.o: $(LIB_DIR)/sphttp.c
-	cc -O2 -c $< -o $@
+	cc -O2 -c $(TEP_SPHTTP_CFLAGS) $< -o $@
 
 $(LIB_DIR)/tep_sqlite.o: $(LIB_DIR)/tep_sqlite.c
 	cc -O2 -c $< -o $@

data/README.md

@@ -4,37 +4,41 @@
 
 # Tep
 
-A Sinatra-flavoured web framework that compiles to a native binary
-via [Spinel][spinel].
+A Sinatra-flavoured web framework that compiles to a **native binary**
+via [Spinel][spinel]. You write a Sinatra-style `app.rb`:
 
-> **Current release:** [v0.11.0](https://github.com/OriPekelman/tep/releases/tag/v0.11.0)
-> — now on [RubyGems](https://rubygems.org/gems/tep) (`gem install tep`):
-> TLS, HTTP caching, connection pooling, PG raise-on-error + embeddings,
-> on top of the agentic surface (Auth, Broadcast, Presence, LiveView, MCP).
+```ruby
+require 'sinatra'
+
+get '/hi/:name' do
+  "hi, " + params[:name] + "!"
+end
+```
+
+…and get `./app` — a single static executable, ~80 KB, **no Ruby
+runtime**, doing ~150k req/s. Beyond routing / sessions / templates, the
+[batteries ↓](#whats-in-the-box) cover auth (bearer / cookie / OAuth2),
+SQLite, opt-in PostgreSQL, WebSockets, Broadcast + Presence + LiveView,
+an MCP tool catalog, TLS, a pooled HTTP client, and an OpenAI-compatible
+LLM client + server.
+
+> **Current release:** [v0.11.6](https://github.com/OriPekelman/tep/releases/tag/v0.11.6)
+> on [RubyGems](https://rubygems.org/gems/tep) — `gem install tep`.
 > Pre-alpha; API still in motion.
 
-> **Why Tep exists.** Two complementary goals:
->
-> 1. **Exercise Spinel against real-world Ruby code.** Tep is the
->    largest pure-Ruby application Spinel compiles end-to-end. Every
->    Sinatra idiom, every battery, every demo doubles as a torture
->    test for the AOT compiler's codegen + analyzer. Bugs surface
->    here, get reduced to minimal repros, and land upstream as PRs
->    or issues against [matz/spinel](https://github.com/matz/spinel).
->    If something doesn't work in Tep, the bug is usually in Spinel,
->    and that's the point.
->
-> 2. **Be the harness for [toy][toy].** Toy is Tep's sibling
->    project: a machine-learning framework written in pure Ruby and
->    compiled by Spinel. Toy needs an HTTP/MCP layer for serving
->    models, exposing training tools to agents (Claude Code et al.),
->    streaming inference results, and wiring presence into
->    collaborative training sessions. Tep is that layer. Every
->    battery in Tep is shaped by what Toy actually needs to ship.
->
-> Tep happens to be useful as a general web framework too — fast,
-> single-binary, Sinatra-shaped — but the design choices are made
-> through these two lenses.
+<details>
+<summary><b>Why Tep exists</b> — it's a deliberate Spinel torture test + toy's serving layer</summary>
+
+Tep is the largest pure-Ruby application Spinel compiles end-to-end:
+every Sinatra idiom and battery doubles as a torture test for the AOT
+compiler's codegen + analyzer, and bugs found here get reduced to
+minimal repros and land upstream as
+[matz/spinel](https://github.com/matz/spinel) PRs. It's also the
+HTTP / MCP / serving layer for its sibling [toy][toy] — a pure-Ruby ML
+framework Spinel compiles — so every battery is shaped by what toy
+needs to ship. Tep is a genuinely useful general web framework too, but
+the design choices are made through those two lenses.
+</details>
 
 [toy]: https://github.com/OriPekelman/toy
 
@@ -107,7 +111,7 @@ gem install. Recommended Ruby manager:
 [`rv`](https://github.com/spinel-coop/rv) — fast version+gem
 manager from the Spinel Cooperative (separate project from the
 matz/spinel AOT compiler Tep compiles through; same Ruby
-neighbourhood). `.ruby-version` in this repo pins 3.4.0; `rv
+neighbourhood). `.ruby-version` in this repo pins 4.0.0; `rv
 shell` makes `rv run rake test` just work. Build deps on Linux:
 `build-essential`, `libsqlite3-dev`. macOS: Xcode CLI tools.
 
@@ -174,7 +178,7 @@ through Spinel.
 | `Tep::WebSocket` | RFC 6455 server-side WebSocket. `websocket '/chat' do \|ws\| ... end` DSL lowers to Frame + Handshake + Driver + Connection. Requires `set :scheduler, :scheduled`. |
 | `Tep::Parallel`  | grosser/parallel-shaped fork fan-out. |
 | `Tep::Job`       | sidekiq-shaped queue over SQLite. |
-| `PG`             | ruby-pg-shape libpq client: `PG::Connection`, `PG::Result`, `PG::Error`; surface mirrors the `pg` gem (`exec` / `exec_params` / `escape_*` / `fields` / `values` / `getvalue` / `sql_state`). Designed so an eventual ActiveRecord-on-spinel port reuses the existing AR adapter with minimal divergence — see `docs/PG-BATTERY.md`. |
+| `PG`             | **Opt-in** ruby-pg-shape libpq client — `require "tep/pg"` (so non-PG apps don't link libpq). `PG::Connection`, `PG::Result`, `PG::Error`; surface mirrors the `pg` gem (`exec` / `exec_params` / `escape_*` / `fields` / `values` / `getvalue` / `sql_state`). Designed so an eventual ActiveRecord-on-spinel port reuses the existing AR adapter with minimal divergence — see `docs/PG-BATTERY.md`. |
 | `Tep::Auth`      | Principal+delegate identity (`Tep::Identity` / `Tep::AgentDelegation`) + provider chain. Three providers shipped: `Tep::AuthBearerToken` (JWT-HS256), `Tep::AuthSessionCookie` (signed cookie), `Tep::AuthOAuth2` (authorization-code grant issuance for bots/agents). Same `req.identity` surface regardless of provider; agents are first-class (`identity.agent?`, `identity.acting_via.agent_id`, capability subsetting). |
 | `Tep::Broadcast` | In-process pub-sub + cross-worker via PG LISTEN/NOTIFY. Subscribe an fd to a topic (`subscribe` raw, `subscribe_ws` WS-frame-wrapped); publish writes to every matching subscriber. The seam Presence and LiveView build on. |
 | `Tep::Presence`  | Topic-keyed who's-here registry, agent-aware. `Tep::Presence.track(req, topic, fd)` records a (principal, session, topic) tuple with a 3-state structured status (`:available | :busy | :blocked` + free-text note + expiry). Diffs broadcast on join/leave/status; PG-mirror for cross-worker `list_global` snapshots. |

data/bin/tep

@@ -1366,6 +1366,30 @@ def handle_top_call(node, routes, websockets, mcp_tools, mcp_resources, filters,
 
   case name
   when "require", "require_relative"
+    # tep/pg is OPT-IN (#216): it is NOT part of the wholesale-inlined
+    # core (lib/tep.rb no longer require_relative's it). An app, test, or
+    # example that needs PostgreSQL says `require "tep/pg"` (or, by path,
+    # `require_relative ".../lib/tep/pg"`); we splice lib/tep/pg.rb in
+    # once here. A non-PG app never names it, so the whole pg closure
+    # (the 74 tep_pg_* FFI symbols + libpq) DCEs away. This must run
+    # BEFORE the is_tep_lib no-op below, which would otherwise swallow
+    # any "tep/..." spelling as already-inlined.
+    pg_arg = node.arguments&.arguments&.first
+    if pg_arg.is_a?(Prism::StringNode) &&
+       (pg_arg.unescaped == "tep/pg" || pg_arg.unescaped.end_with?("/tep/pg"))
+      pgpath = File.join(LIB_DIR, "tep", "pg.rb")
+      if inlined_seen && File.file?(pgpath) && !inlined_seen[pgpath]
+        pgtext = inline_require_relative_tree(pgpath, inlined_seen, warnings)
+        # Resolve pg.rb's @TEP_PG_O@ / @TEP_PG_CFLAGS@ placeholders the
+        # same way inlined_tep_library does for the wholesale core -- the
+        # opt-in splice bypasses that pass, so without this the literal
+        # placeholders reach the linker ("ld: cannot find @TEP_PG_O@").
+        tep_ext_subs.each { |placeholder, value| pgtext = pgtext.gsub(placeholder, value) }
+        passthrough << "# --- inlined opt-in require #{pg_arg.unescaped.inspect} (pg.rb) ---"
+        passthrough << pgtext
+      end
+      return
+    end
     # tep's own library is inlined wholesale (see inlined_tep_library), so
     # `require_relative "tep"` / "tep/..." is a no-op. A plain `require
     # "gem"` (by name) is dropped -- spinel has no gem load path. But an
@@ -1586,7 +1610,14 @@ def rewrite_block(src, force_string: true)
     # are handled separately so they can contain commas or spaces.
     pairs = $2.scan(/(\w+):\s*("[^"]*"|'[^']*'|[^,}]+?)\s*(?=,|\z)/)
     setters = pairs.map { |k, v| %{__l["#{k}"] = (#{v.strip}).to_s} }.join("; ")
-    "(__l = Tep.str_hash; #{setters}; tep_view_#{view}(__l, req.ivars))"
+    # Build the locals hash INSIDE the first argument (a sequence
+    # expression that returns __l), rather than `(__l = ...; render(__l))`
+    # with __l a bare arg. In the latter shape spinel (recent master)
+    # hoists the `render`'s arg temp -- `_t = __l` -- above the in-sequence
+    # `__l = str_hash()` assignment, so render receives the pre-assignment
+    # (empty) hash and every interpolated value blanks (matz/spinel#1478).
+    # Keeping the assignment inside the arg evaluation defeats the hoist.
+    "tep_view_#{view}((__l = Tep.str_hash; #{setters}; __l), req.ivars)"
   end
   s = s.gsub(/(?<![\w.])erb\s+:(\w+)/, 'tep_view_\1(Tep.str_hash, req.ivars)')
 
@@ -1595,7 +1626,8 @@ def rewrite_block(src, force_string: true)
     view = $1
     pairs = $2.scan(/(\w+):\s*("[^"]*"|'[^']*'|[^,}]+?)\s*(?=,|\z)/)
     setters = pairs.map { |k, v| %{__l["#{k}"] = (#{v.strip}).to_s} }.join("; ")
-    "(__l = Tep.str_hash; #{setters}; tep_mustache_#{view}(__l, req.ivars))"
+    # Same arg-hoist avoidance as the erb form above (matz/spinel#1478).
+    "tep_mustache_#{view}((__l = Tep.str_hash; #{setters}; __l), req.ivars)"
   end
   s = s.gsub(/(?<![\w.])mustache\s+:(\w+)/, 'tep_mustache_\1(Tep.str_hash, req.ivars)')
 
@@ -1947,7 +1979,11 @@ def tep_ext_subs
     else
       cflags       = ENV.fetch(env_var, "")
       libs_var     = env_var.sub(/_CFLAGS\z/, "_LIBS")
-      default_libs = entry["pkg_config"] ? "-l" + entry["pkg_config"].sub(/\Alib/, "") : ""
+      # Prefer an explicit pkg_config_fallback (e.g. openssl -> "-lssl
+      # -lcrypto"); the bare "-l<pkg_config>" derivation is only right when
+      # the lib name matches the .pc name (libpq -> -lpq), not for openssl.
+      default_libs = entry["pkg_config_fallback"] ||
+                     (entry["pkg_config"] ? "-l" + entry["pkg_config"].sub(/\Alib/, "") : "")
       libs         = ENV.fetch(libs_var, default_libs)
       subs[placeholder] = "#{cflags} #{libs}".strip
     end
@@ -1979,7 +2015,16 @@ def resolve_ext_o(entry, entries, env_var)
   entries.each do |sib|
     next unless sib["name"] == entry["name"] && sib["pkg_config"]
     pc = `pkg-config --cflags #{sib["pkg_config"]} 2>/dev/null`
-    $?.success? ? cflags.concat(pc.split) : (missing = sib["pkg_config"])
+    if $?.success?
+      cflags.concat(pc.split)
+    elsif sib["pkg_config_fallback"]
+      # pkg-config can't find a .pc (e.g. openssl on a host without it):
+      # assume the headers are on the default include path -- the
+      # historical behavior before this sibling existed -- and let the
+      # link use the fallback libs. Don't fail an otherwise-fine compile.
+    else
+      missing = sib["pkg_config"]
+    end
   end
   if missing
     return "" if entry["optional"]

data/examples/pg_hello.rb

@@ -16,6 +16,7 @@
 # today (matz/spinel#627). Once that lands, this example collapses
 # to the AR-shape `rescue PG::Error => e`.
 require_relative "../lib/tep"
+require_relative "../lib/tep/pg"   # opt-in PG backend (#216)
 
 PG_URL = ENV["PG_URL"] != nil && ENV["PG_URL"].length > 0 ? ENV["PG_URL"] : "postgresql:///postgres"
 
@@ -67,7 +68,17 @@ get '/error' do
     out = "rescued PG::UndefinedTable\n" +
           "sqlstate: " + c.last_sqlstate + "\n" +
           "is undefined-table? " + (c.last_sqlstate == "42P01" ? "yes" : "no") + "\n" +
-          "is PG::Error? " + (e.is_a?(PG::Error) ? "yes" : "no") + "\n" +
+          # WORKAROUND -- still open at SPINEL_PIN (re-checked at the
+          # ad2b71ad re-pin: `e.is_a?(PG::Error)` here is rejected as
+          # `unsupported call: is_a? recv=LocalVariableRead argc=1`).
+          # `e` is the rescued exception, typed PG::UndefinedTable -- a
+          # whole-program is_a? against the namespaced ancestor PG::Error
+          # isn't lowered yet. Minimal `rescue Sub => e; e.is_a?(Super)`
+          # compiles fine; only the full program trips it. Since `e` is
+          # always a PG::Error subclass here, hardcode "yes". Restore
+          #   (e.is_a?(PG::Error) ? "yes" : "no")
+          # once is_a?-on-rescued-namespaced-ancestor lowers. See tep#196.
+          "is PG::Error? " + "yes" + "\n" +
           "message: " + e.message
   end
   c.close

data/lib/tep/broadcast.rb

@@ -114,14 +114,21 @@ module Tep
     # remote deliveries are best-effort and not counted here.
     def self.publish(topic, payload)
       matched = Tep::Broadcast.publish_local_only(topic, payload)
-      if Tep::APP.broadcast_pg_enabled != 0
-        wire = Tep::Broadcast.encode_wire(topic, payload)
-        Tep::APP.broadcast_pg_conn.notify(
-          Tep::APP.broadcast_pg_channel, wire)
-      end
+      # Cross-worker fan-out is an OPT-IN hook (#216): the core build
+      # has no PG reference, so a non-PG app DCEs the entire libpq
+      # closure. `require "tep/pg"` redefines cross_worker_notify with
+      # the real broadcast_pg_conn.notify (last-definition-wins).
+      Tep::Broadcast.cross_worker_notify(topic, payload)
       matched
     end
 
+    # No-op cross-worker hook. Overridden by lib/tep/pg.rb when the PG
+    # backend is loaded. Keeping the PG NOTIFY out of core is what lets
+    # `Tep::Broadcast.publish` compile without pulling tep_pg_*/libpq.
+    def self.cross_worker_notify(topic, payload)
+      0
+    end
+
     # Total subscription count across all topics. Useful for
     # diagnostics and the v1 test surface.
     def self.subscriber_count
@@ -157,81 +164,12 @@ module Tep
 
     # ---- PG backend (cross-worker pub/sub) ----
     #
-    # Opens a dedicated PG connection and issues `LISTEN <channel>`.
-    # Subsequent publishes NOTIFY this channel too -- other workers
-    # subscribed to the same channel can receive the message via
-    # poll_pg_once.
-    #
-    # `conninfo` is the libpq connect string. `channel` must be a
-    # safe SQL identifier (e.g. "tep_broadcast") since it lands
-    # inside a LISTEN / NOTIFY command unescaped.
-    #
-    # Returns 0 on success, -1 on connection or LISTEN failure.
-    def self.enable_pg_backend(conninfo, channel)
-      conn = PG::Connection.new(conninfo)
-      if conn.pgh < 0
-        return -1
-      end
-      if conn.listen(channel) < 0
-        return -1
-      end
-      Tep::APP.set_broadcast_pg_conn(conn)
-      Tep::APP.set_broadcast_pg_channel(channel)
-      Tep::APP.set_broadcast_pg_enabled(1)
-      0
-    end
-
-    def self.disable_pg_backend
-      if Tep::APP.broadcast_pg_enabled == 0
-        return 0
-      end
-      Tep::APP.broadcast_pg_conn.unlisten(Tep::APP.broadcast_pg_channel)
-      Tep::APP.broadcast_pg_conn.finish
-      Tep::APP.set_broadcast_pg_enabled(0)
-      0
-    end
-
-    # Process one notification from the PG channel: parse the wire
-    # format, dispatch to local subscribers as if `publish` had
-    # been called locally (but WITHOUT re-NOTIFYing -- that would
-    # loop). Returns 1 if a notification was processed, 0 on
-    # timeout, -1 on connection error or unenabled backend.
-    def self.poll_pg_once(timeout_ms)
-      if Tep::APP.broadcast_pg_enabled == 0
-        return -1
-      end
-      r = Tep::APP.broadcast_pg_conn.poll_notification(timeout_ms)
-      if r != 1
-        return r
-      end
-      wire = Tep::APP.broadcast_pg_conn.last_notify_payload
-      Tep::Broadcast.deliver_wire_local(wire)
-      1
-    end
-
-    # Wire format: "<topic_byte_length>:<topic><payload>".
-    # Length-prefixed so topics and payloads with arbitrary chars
-    # (commas, colons, embedded quotes, newlines) round-trip
-    # unambiguously. Encoded by `publish` when the PG backend is
-    # enabled; decoded by `deliver_wire_local`.
-    def self.encode_wire(topic, payload)
-      topic.length.to_s + ":" + topic + payload
-    end
-
-    def self.deliver_wire_local(wire)
-      colon = Tep.str_find(wire, ":", 0)
-      if colon <= 0
-        return -1
-      end
-      len_str = wire[0, colon]
-      tlen    = len_str.to_i
-      if tlen < 0 || colon + 1 + tlen > wire.length
-        return -1
-      end
-      topic   = wire[colon + 1, tlen]
-      payload = wire[colon + 1 + tlen, wire.length - colon - 1 - tlen]
-      Tep::Broadcast.publish_local_only(topic, payload)
-    end
+    # The PG LISTEN/NOTIFY backend is OPT-IN (#216). enable_pg_backend /
+    # disable_pg_backend / poll_pg_once, plus the wire encode/decode
+    # helpers (encode_wire / deliver_wire_local) and the
+    # cross_worker_notify override, live in lib/tep/pg.rb and only
+    # compile into apps that `require "tep/pg"`. Core Broadcast carries
+    # no PG reference so a non-PG app DCEs the libpq closure entirely.
 
     # Same fan-out as #publish but skips the PG NOTIFY step. Used
     # internally by poll_pg_once when delivering a cross-worker

data/lib/tep/net.rb

@@ -12,9 +12,14 @@ module Sock
   # libssl/libcrypto. Linked for every app (like sqlite3 elsewhere);
   # the plaintext path never calls into it, so apps that make no HTTPS
   # requests pay only the link cost, not runtime. See tep#148.
-  # (When OpenSSL is off the default path -- macOS/Homebrew -- the build
-  # finds it via CPATH/LIBRARY_PATH in the environment, not a cflag
-  # here; spinel's ffi_cflags rejects an empty-string placeholder.)
+  #
+  # OpenSSL include/lib paths come via @TEP_SPHTTP_CFLAGS@ (the
+  # pkg_config sibling in spinel-ext.json -- `pkg-config openssl`,
+  # fallback `-lssl -lcrypto`), mirroring @TEP_PG_CFLAGS@. On Linux it's
+  # often just the libs (headers on the default path); on macOS/Homebrew
+  # it supplies the keg-only -I/-L too, so sphttp.c compiles + the
+  # ffi_lib "ssl"/"crypto" below resolve. See tep#208.
+  ffi_cflags "@TEP_SPHTTP_CFLAGS@"
   ffi_lib "ssl"
   ffi_lib "crypto"