tep 0.11.3 → 0.11.5

data/examples/llm_gateway/app.rb

@@ -53,7 +53,7 @@ gw.before do |req, res, ureq|
 end
 
 # Stream when the client asked for it. OpenAI signals streaming with
-# `"stream": true` in the JSON body; Tep::Json has no bool getter, so
+# `"stream": true` in the JSON body; SpinelKit::Json has no bool getter, so
 # we match the literal (with or without the space).
 gw.stream_request? do |req|
   b = req.raw_body
@@ -70,15 +70,15 @@ end
 
 # One inference event at end-of-stream -- the right cardinality.
 gw.on_stream_end do |req, out, stats|
-  model = Tep::Json.get_str(req.raw_body, "model")
+  model = SpinelKit::Json.get_str(req.raw_body, "model")
   t0    = req.ivars["t0"].to_i
   wall  = Time.now.to_i - t0
   if wall < 0
     wall = 0
   end
   extra = "{" +
-    Tep::Json.encode_pair_str("request_id", req.req_headers["x-request-id"]) + "," +
-    Tep::Json.encode_pair_str("principal_id", req.identity.principal_id) +
+    SpinelKit::Json.encode_pair_str("request_id", req.req_headers["x-request-id"]) + "," +
+    SpinelKit::Json.encode_pair_str("principal_id", req.identity.principal_id) +
   "}"
   # prompt_tokens unknown at the proxy (no tokenizer); completion_tokens
   # approximated by the SSE event count. wall_us is second-resolution

data/examples/pg_hello.rb

@@ -16,6 +16,7 @@
 # today (matz/spinel#627). Once that lands, this example collapses
 # to the AR-shape `rescue PG::Error => e`.
 require_relative "../lib/tep"
+require_relative "../lib/tep/pg"   # opt-in PG backend (#216)
 
 PG_URL = ENV["PG_URL"] != nil && ENV["PG_URL"].length > 0 ? ENV["PG_URL"] : "postgresql:///postgres"
 
@@ -67,7 +68,16 @@ get '/error' do
     out = "rescued PG::UndefinedTable\n" +
           "sqlstate: " + c.last_sqlstate + "\n" +
           "is undefined-table? " + (c.last_sqlstate == "42P01" ? "yes" : "no") + "\n" +
-          "is PG::Error? " + (e.is_a?(PG::Error) ? "yes" : "no") + "\n" +
+          # WORKAROUND -- REMOVE WHEN UPSTREAM LANDS: `e.is_a?(PG::Error)`
+          # miscompiles at spinel master (whole-program is_a? on a deep
+          # exception subclass vs an ancestor -- e here is rescued as
+          # PG::UndefinedTable, a PG::Error subclass, so this is always
+          # "yes"). Minimal `rescue Sub => e; e.is_a?(Super)` repros
+          # compile fine; it only trips in the full program. Hardcoded
+          # to "yes" to keep the example building for the re-pin. See
+          # matz/spinel#1434, tep#196. Original:
+          #   (e.is_a?(PG::Error) ? "yes" : "no")
+          "is PG::Error? " + "yes" + "\n" +
           "message: " + e.message
   end
   c.close

data/lib/spinel_kit/hex.rb

@@ -0,0 +1,65 @@
+# VENDORED from OriPekelman/spinelkit @ 09e8558 -- DO NOT EDIT HERE.
+# Edit upstream and re-sync with `make vendor-spinelkit`.
+# SpinelKit::Hex -- hex digit/byte encode + decode: the pieces every Spinel
+# project re-rolls. The decode nibble appeared BYTE-IDENTICAL in Tep::Url,
+# inside SpinelKit::Json's string decoder, and (as a multi-digit variant) in
+# Tep::Llm's chunked-transfer size parser. Pure string/byte ops, Spinel-safe.
+#
+# NOTE on the Json overlap: SpinelKit::Json keeps its OWN private `hex2`/
+# `hex_nibble` so a JSON-only consumer never compiles this file (Spinel has no
+# tree-shaking — see json.rb). Hex is the shared surface for everyone else,
+# e.g. SpinelKit::Url.
+module SpinelKit
+  class Hex
+    # Hex digit char -> int 0..15, or -1 if not a hex digit (upper or lower).
+    def self.nibble(c)
+      if c >= "0" && c <= "9"
+        return c.getbyte(0) - "0".getbyte(0)
+      end
+      if c >= "a" && c <= "f"
+        return c.getbyte(0) - "a".getbyte(0) + 10
+      end
+      if c >= "A" && c <= "F"
+        return c.getbyte(0) - "A".getbyte(0) + 10
+      end
+      -1
+    end
+
+    # Int nibble 0..15 -> single UPPERCASE hex char ("0".."9","A".."F").
+    # (RFC 3986 percent-encoding uses uppercase.)
+    def self.nibble_char(n)
+      if n < 10
+        return ("0".getbyte(0) + n).chr
+      end
+      ("A".getbyte(0) + n - 10).chr
+    end
+
+    # Int byte 0..255 -> two-char LOWERCASE hex (15 -> "0f"). (JSON \u00XX
+    # and similar use lowercase.)
+    def self.byte2(n)
+      hex = "0123456789abcdef"
+      out = ""
+      out = out + hex[(n / 16) % 16, 1]
+      out = out + hex[n % 16, 1]
+      out
+    end
+
+    # Parse the leading hex digits of `s` -> int ("1a3" -> 419). Stops at the
+    # first non-hex char; returns 0 if there is no leading hex digit. Useful
+    # for chunked-transfer sizes and the like.
+    def self.to_int(s)
+      n = 0
+      i = 0
+      len = s.length
+      while i < len
+        v = Hex.nibble(s[i])
+        if v < 0
+          return n
+        end
+        n = n * 16 + v
+        i += 1
+      end
+      n
+    end
+  end
+end

data/lib/spinel_kit/json.rb

@@ -0,0 +1,151 @@
+# VENDORED from OriPekelman/spinelkit @ 09e8558 -- DO NOT EDIT HERE.
+# Edit upstream and re-sync with `make vendor-spinelkit`.
+# SpinelKit::Json -- Spinel-safe JSON ENCODERS (stateless).
+#
+# This file holds the encode half of the codec; the decode half lives in
+# spinel_kit/json_decoder.rb (also `SpinelKit::Json`), and the incremental
+# object builder in spinel_kit/json_builder.rb (`SpinelKit::Json::Builder`).
+# The three are split because Spinel has no tree-shaking: every loaded method
+# is compiled, and a set of uncalled methods can degrade each other's params
+# (e.g. the dead decoder walkers collectively widening `escape`'s string arg
+# to int, which silently miscompiled string keys to ""). Keeping
+# encode/decode/build in separate files means a consumer compiles only the
+# surface it calls, and each surface is independently warning-clean. Require
+# only what you use:
+#
+#   require "spinel_kit/json"          # encoders (this file)
+#   require "spinel_kit/json_decoder"  # decoders
+#   require "spinel_kit/json_builder"  # builder
+#
+# WHY HAND-ROLLED. Spinel cannot lower the stdlib `json` gem (C-ext fast path
+# + metaprogrammed pure fallback); `oj`/`yajl`/`multi_json` are C extensions.
+# The spinelgems catalog confirms no verified pure-Ruby JSON gem exists. This
+# is tep's encoder, standardized (the `j_`/`tj_` prefixes that worked around a
+# now-fixed Spinel inference bug are gone -- see docs/spinel-discipline.md).
+#
+# Compose objects in user code by concatenation:
+#
+#   "{" + SpinelKit::Json.encode_pair_str("name", name) + "," +
+#         SpinelKit::Json.encode_pair_int("age", age) + "}"
+module SpinelKit
+  class Json
+    # Escape a string for inclusion inside a JSON string literal (does NOT
+    # add the surrounding quotes -- use `quote(s)` for that). Handles ", \,
+    # and the JSON-required control-char escapes (\b, \f, \n, \r, \t);
+    # other control bytes go through \u00XX. Forward slash is left
+    # unescaped (legal either way; unescaped is shorter/readable).
+    def self.escape(s)
+      out = ""
+      i = 0
+      n = s.length
+      while i < n
+        c = s[i]
+        if c == "\""
+          out = out + "\\\""
+        elsif c == "\\"
+          out = out + "\\\\"
+        elsif c == "\n"
+          out = out + "\\n"
+        elsif c == "\r"
+          out = out + "\\r"
+        elsif c == "\t"
+          out = out + "\\t"
+        elsif c == "\b"
+          out = out + "\\b"
+        elsif c == "\f"
+          out = out + "\\f"
+        elsif c < " "
+          # Other control byte -- emit \u00XX. c.getbyte(0) is the raw
+          # byte value, mapped to two hex digits.
+          b = c.getbyte(0)
+          out = out + "\\u00" + Json.hex2(b)
+        else
+          out = out + c
+        end
+        i += 1
+      end
+      out
+    end
+
+    # Two-digit lowercase hex of a byte (0..255).
+    def self.hex2(n)
+      hex = "0123456789abcdef"
+      out = ""
+      out = out + hex[(n / 16) % 16, 1]
+      out = out + hex[n % 16, 1]
+      out
+    end
+
+    # Wrap a string in JSON quotes, escaping its body.
+    def self.quote(s)
+      "\"" + Json.escape(s) + "\""
+    end
+
+    # Encode a single key/value pair as `"k":"v"` (escaped both sides).
+    def self.encode_pair_str(k, v)
+      Json.quote(k) + ":" + Json.quote(v)
+    end
+
+    # Same shape, integer value side. `v` is rendered via `.to_s` so
+    # JSON-numeric output without quoting.
+    def self.encode_pair_int(k, v)
+      Json.quote(k) + ":" + v.to_s
+    end
+
+    # Encode a Hash<String,String> as a JSON object.
+    def self.from_str_hash(h)
+      out = "{"
+      first = true
+      h.each do |k, v|
+        if !first
+          out = out + ","
+        end
+        first = false
+        out = out + Json.quote(k) + ":" + Json.quote(v)
+      end
+      out + "}"
+    end
+
+    # Same shape with integer values. JSON-numeric, no quoting.
+    def self.from_int_hash(h)
+      out = "{"
+      first = true
+      h.each do |k, v|
+        if !first
+          out = out + ","
+        end
+        first = false
+        out = out + Json.quote(k) + ":" + v.to_s
+      end
+      out + "}"
+    end
+
+    # Encode a string array as a JSON array of quoted strings.
+    def self.from_str_array(a)
+      out = "["
+      i = 0
+      while i < a.length
+        if i > 0
+          out = out + ","
+        end
+        out = out + Json.quote(a[i])
+        i += 1
+      end
+      out + "]"
+    end
+
+    # Encode an int array as a JSON array of numbers.
+    def self.from_int_array(a)
+      out = "["
+      i = 0
+      while i < a.length
+        if i > 0
+          out = out + ","
+        end
+        out = out + a[i].to_s
+        i += 1
+      end
+      out + "]"
+    end
+  end
+end

data/lib/spinel_kit/json_decoder.rb

@@ -0,0 +1,396 @@
+# VENDORED from OriPekelman/spinelkit @ 09e8558 -- DO NOT EDIT HERE.
+# Edit upstream and re-sync with `make vendor-spinelkit`.
+# SpinelKit::Json -- Spinel-safe JSON DECODERS (flat-key, top-level only).
+#
+# The decode half of the codec; encoders are in spinel_kit/json.rb. Split out
+# so an encode-only consumer never compiles these walkers (their dead-code
+# degradation otherwise widens the encoders' string args to int -- see the
+# header of json.rb and docs/spinel-discipline.md).
+#
+# `get_str(s, key)` finds the entry for `key` in the top-level object literal
+# `s` and returns its value as a string. Returns "" when `key` is absent or
+# the value isn't a string. Same shape for `get_int`. `has_key?(s, key)`
+# returns a boolean independent of value type. The parser is a hand-rolled
+# state machine that walks one `{ "k": <value>, ... }` pair at a time,
+# skipping over any value (including nested objects / arrays) it doesn't need.
+# Strings inside values are honoured for escape sequences so that `\"` doesn't
+# terminate the string and corrupt the walk. Decodes the escape sequences
+# `SpinelKit::Json.escape` produces.
+module SpinelKit
+  class Json
+    def self.get_str(s, key)
+      pos = Json.find_value_start(s, key)
+      if pos < 0
+        return ""
+      end
+      Json.parse_str_value(s, pos)
+    end
+
+    def self.get_int(s, key)
+      pos = Json.find_value_start(s, key)
+      if pos < 0
+        return 0
+      end
+      Json.parse_int_value(s, pos)
+    end
+
+    # Decode a JSON number value at `key` -> Float. Accepts both
+    # integer-literal (`42`) and float-literal (`3.14`, `-0.5`, `1e2`)
+    # JSON-number syntax; the integer form returns N.0. Missing key or
+    # malformed value returns 0.0 (consistent with the other getters'
+    # missing-key defaults).
+    #
+    # Implementation: delegates the value-span walking to skip_value (already
+    # handles all JSON-number syntax + structural-char boundaries), then
+    # String#to_f on the substring. Inlined rather than factored into a
+    # parse_float_value helper because spinel's type inference mis-widens `s`
+    # to int through the indirection. NOTE: that is a value-walk indirection
+    # concern, NOT the name-collision bug (which was fixed) -- keep it inlined.
+    def self.get_float(s, key)
+      pos = Json.find_value_start(s, key)
+      if pos < 0
+        return 0.0
+      end
+      pos = Json.skip_ws(s, pos)
+      if pos >= s.length
+        return 0.0
+      end
+      end_pos = Json.skip_value(s, pos)
+      if end_pos <= pos
+        return 0.0
+      end
+      s[pos, end_pos - pos].to_f
+    end
+
+    def self.has_key?(s, key)
+      Json.find_value_start(s, key) >= 0
+    end
+
+    # Decode a flat JSON array of integers at `key` -> Array[Integer].
+    # A missing or non-array value yields [] (the typed-empty-array idiom);
+    # non-int elements are skipped.
+    def self.get_int_array(s, key)
+      out = [0]
+      out.delete_at(0)
+      pos = Json.find_value_start(s, key)
+      if pos < 0
+        return out
+      end
+      pos = Json.skip_ws(s, pos)
+      if pos >= s.length || s[pos] != "["
+        return out
+      end
+      pos += 1
+      while pos < s.length
+        pos = Json.skip_ws(s, pos)
+        if pos >= s.length
+          return out
+        end
+        c = s[pos]
+        if c == "]"
+          return out
+        elsif c == ","
+          pos += 1
+        elsif (c >= "0" && c <= "9") || c == "-"
+          out.push(Json.parse_int_value(s, pos))
+          # Advance past the number parse_int_value just consumed
+          # (optional '-' then digits).
+          if s[pos] == "-"
+            pos += 1
+          end
+          while pos < s.length && s[pos] >= "0" && s[pos] <= "9"
+            pos += 1
+          end
+        else
+          # Non-int element (string / object / etc.): skip it.
+          pos = Json.skip_value(s, pos)
+        end
+      end
+      out
+    end
+
+    # ---- Internal helpers ----
+
+    # Skip whitespace starting at `pos`, return the new position.
+    def self.skip_ws(s, pos)
+      while pos < s.length
+        c = s[pos]
+        if c == " " || c == "\t" || c == "\n" || c == "\r"
+          pos += 1
+        else
+          return pos
+        end
+      end
+      pos
+    end
+
+    # Walk a JSON-quoted string starting at `pos` (which must point at the
+    # opening `"`). Returns the position one past the closing `"`. Returns
+    # -1 on malformed input.
+    def self.skip_str(s, pos)
+      if pos >= s.length || s[pos] != "\""
+        return -1
+      end
+      pos += 1
+      while pos < s.length
+        c = s[pos]
+        if c == "\\"
+          # Skip the escape and the escaped character. \uXXXX spans 6
+          # chars total but skipping 2 still keeps us inside the string
+          # for the rest of the walk -- the remaining 4 hex digits look
+          # like ordinary string bytes and won't terminate the literal.
+          pos += 2
+        elsif c == "\""
+          return pos + 1
+        else
+          pos += 1
+        end
+      end
+      -1
+    end
+
+    # Walk a JSON value starting at `pos` (which must point at the first
+    # non-ws char of the value). Returns the position one past the value
+    # (or the input length on truncation).
+    def self.skip_value(s, pos)
+      pos = Json.skip_ws(s, pos)
+      if pos >= s.length
+        return pos
+      end
+      c = s[pos]
+      if c == "\""
+        return Json.skip_str(s, pos)
+      end
+      if c == "{" || c == "["
+        return Json.skip_container(s, pos)
+      end
+      # number / true / false / null -- read until the next structural /
+      # whitespace char.
+      while pos < s.length
+        c = s[pos]
+        if c == "," || c == "}" || c == "]" ||
+           c == " " || c == "\t" || c == "\n" || c == "\r"
+          return pos
+        end
+        pos += 1
+      end
+      pos
+    end
+
+    # Walk a balanced { ... } or [ ... ] starting at `pos`. Honours string
+    # literals so that `{` / `}` inside a value-string don't confuse the
+    # brace counter. Returns position one past the matching closer.
+    def self.skip_container(s, pos)
+      open_c = s[pos]
+      close_c = open_c == "{" ? "}" : "]"
+      depth = 1
+      pos += 1
+      while pos < s.length && depth > 0
+        c = s[pos]
+        if c == "\""
+          # whole nested string -- skip past it
+          npos = Json.skip_str(s, pos)
+          if npos < 0
+            return s.length
+          end
+          pos = npos
+        elsif c == open_c
+          depth += 1
+          pos += 1
+        elsif c == close_c
+          depth -= 1
+          pos += 1
+        else
+          pos += 1
+        end
+      end
+      pos
+    end
+
+    # Read a JSON-quoted string at `pos` and return its decoded contents
+    # (no surrounding quotes). Decodes the same escape sequences that
+    # `escape` produces. Returns "" on malformed input.
+    def self.parse_str_value(s, pos)
+      pos = Json.skip_ws(s, pos)
+      if pos >= s.length || s[pos] != "\""
+        return ""
+      end
+      pos += 1
+      out = ""
+      while pos < s.length
+        c = s[pos]
+        if c == "\""
+          return out
+        end
+        if c == "\\"
+          if pos + 1 >= s.length
+            return out
+          end
+          esc = s[pos + 1]
+          if esc == "\""
+            out = out + "\""
+          elsif esc == "\\"
+            out = out + "\\"
+          elsif esc == "/"
+            out = out + "/"
+          elsif esc == "n"
+            out = out + "\n"
+          elsif esc == "r"
+            out = out + "\r"
+          elsif esc == "t"
+            out = out + "\t"
+          elsif esc == "b"
+            out = out + "\b"
+          elsif esc == "f"
+            out = out + "\f"
+          elsif esc == "u"
+            # \u00XX -> map the two-digit hex back to a byte. Wider
+            # codepoints (U+0100+ or surrogate pairs) aren't decoded; the
+            # byte we emit is the low byte of the codepoint, which
+            # round-trips ASCII at minimum.
+            if pos + 5 < s.length
+              h1 = Json.hex_nibble(s[pos + 4])
+              h2 = Json.hex_nibble(s[pos + 5])
+              if h1 >= 0 && h2 >= 0
+                # rebuild the byte and push it -- spinel strings are
+                # byte-blobs, so this works for ASCII; for non-ASCII the
+                # original encoder would have used a passthrough byte
+                # anyway.
+                b = h1 * 16 + h2
+                out = out + Json.byte_to_chr(b)
+                pos += 6
+                next
+              end
+            end
+            out = out + "?"
+            pos += 2
+            next
+          else
+            out = out + esc
+          end
+          pos += 2
+        else
+          out = out + c
+          pos += 1
+        end
+      end
+      out
+    end
+
+    def self.hex_nibble(c)
+      if c >= "0" && c <= "9"
+        return c.getbyte(0) - "0".getbyte(0)
+      end
+      if c >= "a" && c <= "f"
+        return c.getbyte(0) - "a".getbyte(0) + 10
+      end
+      if c >= "A" && c <= "F"
+        return c.getbyte(0) - "A".getbyte(0) + 10
+      end
+      -1
+    end
+
+    # Build a single-byte string from an integer 0..255. Spinel doesn't
+    # expose `n.chr` for arbitrary bytes uniformly; the table covers the
+    # ASCII printable range and falls back to "?" for anything else (the
+    # JSON encoder side never produces non-ASCII via \u, so the fallback
+    # is reachable only for malformed input).
+    def self.byte_to_chr(n)
+      printable = " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
+      if n >= 32 && n < 127
+        return printable[n - 32, 1]
+      end
+      if n == 9
+        return "\t"
+      end
+      if n == 10
+        return "\n"
+      end
+      if n == 13
+        return "\r"
+      end
+      "?"
+    end
+
+    # Read an integer at `pos`. Accepts an optional leading `-`. Returns 0
+    # on no-digit / non-numeric input (caller can use `has_key?` first if
+    # 0-vs-absent matters).
+    def self.parse_int_value(s, pos)
+      pos = Json.skip_ws(s, pos)
+      if pos >= s.length
+        return 0
+      end
+      neg = false
+      if s[pos] == "-"
+        neg = true
+        pos += 1
+      end
+      n = 0
+      saw_digit = false
+      while pos < s.length
+        c = s[pos]
+        if c >= "0" && c <= "9"
+          n = n * 10 + (c.getbyte(0) - "0".getbyte(0))
+          saw_digit = true
+          pos += 1
+        else
+          break
+        end
+      end
+      if !saw_digit
+        return 0
+      end
+      neg ? -n : n
+    end
+
+    # Walk the top-level object looking for the entry whose key matches
+    # `target_key`; return the position of the value's first non-ws
+    # character. Returns -1 if not found.
+    def self.find_value_start(s, target_key)
+      pos = Json.skip_ws(s, 0)
+      if pos >= s.length || s[pos] != "{"
+        return -1
+      end
+      pos += 1
+      while pos < s.length
+        pos = Json.skip_ws(s, pos)
+        if pos >= s.length
+          return -1
+        end
+        if s[pos] == "}"
+          return -1
+        end
+        # Read a key.
+        if s[pos] != "\""
+          return -1
+        end
+        key_start = pos
+        pos = Json.skip_str(s, pos)
+        if pos < 0
+          return -1
+        end
+        # Decode the key for comparison (handles \" inside keys).
+        key = Json.parse_str_value(s, key_start)
+        # Skip ws, ":".
+        pos = Json.skip_ws(s, pos)
+        if pos >= s.length || s[pos] != ":"
+          return -1
+        end
+        pos += 1
+        pos = Json.skip_ws(s, pos)
+        if key == target_key
+          return pos
+        end
+        # Skip the value, then the comma (if any).
+        pos = Json.skip_value(s, pos)
+        pos = Json.skip_ws(s, pos)
+        if pos < s.length && s[pos] == ","
+          pos += 1
+        elsif pos < s.length && s[pos] == "}"
+          return -1
+        end
+      end
+      -1
+    end
+  end
+end