ruby_llm-providers-typesafe 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: ed59a9e9d3a05dc3295e783fed03656a1c7991bc7569f6b3f998447d1ff0af13
+  data.tar.gz: 6bf922a4552c10571302adba8c53c05aefe3d684490adf259b0c885c8fd4b140
+SHA512:
+  metadata.gz: 9d4ac70d8ee3fd3f473abffc61d7c586dd2672b7492b1f3c3548e3c9449aa7707cf17f887caa8796550bf36906f8bfc3c633a90c7a47dfc1405fa1ce1bf56285
+  data.tar.gz: f8722a75e6928beba4d2ae6269aa133c580305fd91d6629e4987f487e8795c0bb8c24493de8771097e8995df9f5cf39feedd900f26a9c1be8496d6959be67c02

data/.flayignore

@@ -0,0 +1 @@
+spec/support/streaming_error_helpers.rb

data/.github/workflows/ci.yml

@@ -0,0 +1,32 @@
+name: CI
+
+on:
+  pull_request:
+    branches: ["main"]
+  push:
+    branches: ["main"]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: Ruby ${{ matrix.ruby-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version: ["3.1", "3.2", "3.3", "3.4", "4.0"]
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          bundler-cache: true
+
+      - run: bundle exec rake
+        if: matrix.ruby-version != '3.1'
+
+      - run: bundle exec rake rubocop flay spec
+        if: matrix.ruby-version == '3.1'

data/.github/workflows/gitleaks.yml

@@ -0,0 +1,22 @@
+name: Gitleaks
+
+on:
+  pull_request:
+    branches: ["main"]
+  push:
+    branches: ["main"]
+
+permissions:
+  contents: read
+
+jobs:
+  scan:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - uses: gitleaks/gitleaks-action@v2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

data/.github/workflows/release.yml

@@ -0,0 +1,33 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  publish:
+    name: Build and publish
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: "3.4"
+          bundler-cache: true
+
+      - run: bundle exec rake
+
+      - name: Build gem
+        run: gem build ruby_llm-providers-typesafe.gemspec
+
+      - name: Configure RubyGems trusted publishing
+        uses: rubygems/configure-rubygems-credentials@v2.1.0
+
+      - name: Publish to RubyGems
+        run: gem push ruby_llm-providers-typesafe-*.gem

data/.overcommit.yml

@@ -0,0 +1,31 @@
+PreCommit:
+  RuboCop:
+    enabled: true
+    auto_correct: true
+    on_warn: fail
+
+  Flay:
+    enabled: true
+    include:
+      - lib/**/*.rb
+      - spec/**/*.rb
+    mass_threshold: 70
+
+  ArchSpec:
+    enabled: true
+    command: ['bundle', 'exec', 'archspec', 'check']
+    on_warn: fail
+
+  RSpec:
+    enabled: true
+    command: ['bundle', 'exec', 'rspec']
+    on_warn: fail
+
+  TrailingWhitespace:
+    enabled: true
+    auto_correct: true
+
+  Gitleaks:
+    enabled: true
+    required_executable: gitleaks
+    command: ['gitleaks', 'protect', '--staged', '--redact', '--verbose']

data/.rspec

@@ -0,0 +1,2 @@
+--require spec_helper
+--format progress

data/.rubocop.yml

@@ -0,0 +1,35 @@
+plugins:
+  - rubocop-performance
+  - rubocop-rake
+  - rubocop-rspec
+
+AllCops:
+  NewCops: enable
+  TargetRubyVersion: 3.1
+  Exclude:
+    - Archspec.rb
+    - vendor/**/*
+    - gemfiles/**/*
+  SuggestExtensions: false
+
+Metrics/AbcSize:
+  Max: 80
+Metrics/MethodLength:
+  Max: 40
+  CountAsOne:
+    - array
+    - hash
+    - heredoc
+Metrics/BlockLength:
+  Exclude:
+    - spec/**/*.rb
+RSpec/ExampleLength:
+  Enabled: false
+RSpec/MultipleExpectations:
+  Enabled: false
+RSpec/MultipleMemoizedHelpers:
+  Max: 8
+Naming/MethodParameterName:
+  AllowedNames:
+    - id
+    - 'no'

data/Archspec.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+source 'lib/**/*.rb'
+
+component :provider,
+          in: %w[
+            lib/ruby_llm/providers/typesafe.rb
+            lib/ruby_llm/providers/typesafe/**/*.rb
+          ],
+          namespace: 'RubyLLM::Providers::Typesafe'
+
+provider.cannot_reference_constants 'RSpec', 'WebMock', 'VCR'
+
+preset :ruby_conventions
+
+component :api,
+          in: %w[
+            lib/ruby_llm/typesafe.rb
+            lib/ruby_llm/typesafe/**/*.rb
+          ],
+          namespace: 'RubyLLM::Typesafe'
+
+api.cannot_reference_constants 'RubyLLM::Providers', 'RSpec', 'WebMock', 'VCR'

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Javier Gradiche
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,84 @@
+# ruby_llm-providers-typesafe
+
+[TypeSafe](https://typesafe.ai) for [RubyLLM](https://rubyllm.com). TypeSafe's System One models, starting with Jev, answer typed questions about your data with calibrated probabilities instead of generated text.
+
+## Installation
+
+```ruby
+gem 'ruby_llm-providers-typesafe'
+```
+
+```ruby
+RubyLLM.configure do |config|
+  config.typesafe_api_key = ENV['TYPESAFE_API_KEY']
+end
+```
+
+Outside Bundler, load it with `require 'ruby_llm/providers/typesafe'`.
+
+## Evaluate
+
+Ask questions about a state, and get one typed answer per question:
+
+```ruby
+evaluation = RubyLLM::Typesafe.evaluate("Help! My payouts have been failing for 3 days.") do |q|
+  q.noul :urgent, "Does this convey urgency?"
+  q.choice :team, "Which team should handle this?",
+           billing: "Payments, invoicing, refunds",
+           technical: "Bugs, outages, integrations",
+           sales: "Pricing, upgrades, new accounts"
+  q.score :frustration, "How frustrated is the customer?", ["Calm", "Frustrated", "Very angry"]
+end
+
+evaluation[:urgent].probability    # => 0.92
+evaluation[:urgent].yes?           # => true
+evaluation[:team].option           # => :technical
+evaluation[:team].probabilities    # => {billing: 0.08, technical: 0.85, sales: 0.07}
+evaluation[:team].confidence       # => 0.82
+evaluation[:frustration].score     # => 1.6
+evaluation[:frustration].level     # => "Very angry"
+evaluation.model                   # => "jev-1.13.0"
+evaluation.cost.total              # => 0.0000131
+```
+
+There are three question types:
+
+* `noul` asks a yes/no question. Its answer is the probability of yes. Pass `yes:` and `no:` to describe what each answer means.
+* `choice` picks one option. Pass descriptions as keywords or a Hash, or an Array of options that need none. The answer uses the keys you passed.
+* `score` rates the state on ordered levels, lowest first. The answer can land between levels.
+
+The state can be a String, or a Hash or Array for structured data. Refer to parts of it from your questions with backticked paths such as `` `ticket.messages[0].text` ``. Every question runs in parallel against the same state, so ask independent questions together.
+
+`yes?` compares the probability to 0.5 by default. Choose thresholds from your own data with `yes?(threshold: 0.8)`, and use `confidence` to send uncertain choices and scores to a person. See TypeSafe's guides to [writing questions](https://docs.typesafe.ai/concepts/how-to-build-with-system-one) and [confidence](https://docs.typesafe.ai/confidence).
+
+Evaluations default to `jev-latest`. Pin a version once you have tuned thresholds against it:
+
+```ruby
+RubyLLM::Typesafe.evaluate(ticket, model: "jev-1.13.0") { |q| q.noul :spam, "Is this message spam?" }
+```
+
+## Rerank
+
+Jev also works with `RubyLLM.rerank`. Every document gets a relevance question in a single request, and its score is the probability that it helps answer the query:
+
+```ruby
+rerank = RubyLLM.rerank("What is the capital of the United States?",
+                        ["Carson City is the capital of Nevada.",
+                         "Washington, D.C. is the capital of the United States."],
+                        model: "jev-latest", provider: :typesafe, top_n: 1)
+
+rerank.results.first.document # => "Washington, D.C. is the capital of the United States."
+```
+
+The query and all documents share Jev's 32k-token state budget. Rerank a shortlist from a faster search, not a whole corpus.
+
+## Development
+
+```sh
+bin/setup
+bundle exec rake
+```
+
+Copy `.env.example` to `.env` and set `TYPESAFE_API_KEY` to record VCR cassettes. The first local run calls the API and records them; CI only replays committed cassettes. A failing live example deletes its cassette so the next run hits the API again.
+
+`bundle exec rake models` refreshes `models.json` from TypeSafe's model listing. The listing returns aliases such as `jev-latest`; versioned ids such as `jev-1.13.0` are accepted without being listed.

data/lib/ruby_llm/providers/typesafe/system_one/evaluations.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class Typesafe < Provider
+      class SystemOne < Protocol
+        # Renders Typesafe::Questions to the wire and parses the answers
+        # into a Typesafe::Evaluation.
+        module Evaluations
+          def evaluation_url
+            'v1/systemone'
+          end
+
+          def evaluate(state, questions, model:)
+            response = @connection.post evaluation_url, render_evaluation_payload(state, questions, model:)
+            parse_evaluation_response(response, questions)
+          end
+
+          def render_evaluation_payload(state, questions, model:)
+            {
+              state: state,
+              model: model,
+              questions: questions.to_h.to_h { |id, question| [id.to_s, render_question(question)] }
+            }
+          end
+
+          def render_question(question)
+            rendered = { type: question.type.to_s, instructions: question.instructions }
+            criteria = render_criteria(question)
+            criteria ? rendered.merge(criteria: criteria) : rendered
+          end
+
+          def parse_evaluation_response(response, questions)
+            data = response.body
+            answers = questions.to_h.to_h do |id, question|
+              [id, parse_answer(question, data.dig('answers', id.to_s) || {})]
+            end
+            usage = data['usage'] || {}
+
+            RubyLLM::Typesafe::Evaluation.new(
+              answers: answers,
+              model: data['model'],
+              tokens: Tokens.new(input: usage['input_tokens'], output: usage['output_tokens']),
+              model_info: @model,
+              raw: data
+            )
+          end
+
+          def parse_answer(question, data)
+            case question.type
+            when :noul then RubyLLM::Typesafe::Evaluation::Noul.new(probability: data['noul'])
+            when :choice then parse_choice_answer(question, data)
+            when :score then parse_score_answer(question, data)
+            end
+          end
+
+          private
+
+          def render_criteria(question)
+            case question.type
+            when :noul
+              criteria = { true => question.criteria[:yes], false => question.criteria[:no] }.compact
+              criteria unless criteria.empty?
+            when :choice then question.criteria.to_h { |option, description| [option.to_s, description] }
+            when :score then question.criteria
+            end
+          end
+
+          def parse_choice_answer(question, data)
+            options = question.criteria.keys.to_h { |option| [option.to_s, option] }
+
+            RubyLLM::Typesafe::Evaluation::Choice.new(
+              option: options.fetch(data['choice'], data['choice']),
+              probabilities: (data['probabilities'] || {}).to_h do |option, value|
+                [options.fetch(option, option), value]
+              end,
+              confidence: data['confidence']
+            )
+          end
+
+          def parse_score_answer(question, data)
+            levels = question.criteria
+
+            RubyLLM::Typesafe::Evaluation::Score.new(
+              score: data['score'],
+              levels: levels,
+              probabilities: levels.each_index.map { |index| data.dig('probabilities', index.to_s) },
+              confidence: data['confidence']
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/typesafe/system_one/models.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class Typesafe < Provider
+      class SystemOne < Protocol
+        # Model listing for the System One API. Every model is billed per
+        # input token; output tokens are free.
+        module Models
+          INPUT_PRICE_PER_MILLION = 0.042
+
+          def models_url
+            'v1/models'
+          end
+
+          def parse_list_models_response(response, slug)
+            Array(response.body['models']).map do |model_data|
+              Model.new(
+                id: model_data['name'],
+                name: model_data['name'],
+                provider: slug,
+                created_at: model_data['release_date'],
+                modalities: { input: ['text'], output: ['rerank'] },
+                capabilities: [],
+                pricing: { text_tokens: { standard: { input_per_million: INPUT_PRICE_PER_MILLION,
+                                                      output_per_million: 0 } } },
+                metadata: { description: model_data['description'] }.compact
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/typesafe/system_one/rerank.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class Typesafe < Provider
+      class SystemOne < Protocol
+        # Reranking as one request: the query and documents form the state,
+        # and each document gets its own relevance question. The noul, the
+        # probability that the document is relevant, is its score.
+        module Rerank
+          RELEVANCE_CRITERIA = {
+            true => 'The document contains information that answers or directly addresses the query.',
+            false => 'The document is off topic or only shares words with the query.'
+          }.freeze
+
+          def rerank_url
+            evaluation_url
+          end
+
+          def render_rerank_payload(query, documents, model:, top_n: nil, provider_options: {})
+            @top_n = top_n
+            questions = documents.each_index.to_h do |index|
+              [rerank_question_id(index), { type: 'noul', instructions: relevance_instructions(index),
+                                            criteria: RELEVANCE_CRITERIA }]
+            end
+
+            { state: { query: query, documents: documents }, model: model, questions: questions }
+              .merge(provider_options)
+          end
+
+          def parse_rerank_response(response, model:, documents: [])
+            data = response.body
+            results = documents.each_with_index.map do |document, index|
+              RubyLLM::Rerank::Result.new(index: index, document: document,
+                                          score: data.dig('answers', rerank_question_id(index), 'noul'))
+            end
+            results = results.sort_by { |result| -result.score.to_f }
+            results = results.first(@top_n) if @top_n
+
+            RubyLLM::Rerank.new(results: results, model: data['model'] || model, raw: data,
+                                input_tokens: data.dig('usage', 'input_tokens'))
+          end
+
+          private
+
+          def rerank_question_id(index)
+            "document_#{index}"
+          end
+
+          def relevance_instructions(index)
+            "Does `documents[#{index}]` help answer `query`?"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/typesafe/system_one.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative 'system_one/models'
+require_relative 'system_one/evaluations'
+require_relative 'system_one/rerank'
+
+module RubyLLM
+  module Providers
+    class Typesafe < Provider
+      # The System One wire format: a state and a map of typed questions go
+      # in, one typed answer per question comes back.
+      class SystemOne < Protocol
+        include Models
+        include Evaluations
+        include Rerank
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/typesafe.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'ruby_llm'
+require_relative '../typesafe'
+require_relative 'typesafe/system_one'
+
+module RubyLLM
+  module Providers
+    # TypeSafe API integration. TypeSafe's System One models, such as Jev,
+    # answer typed questions about a state instead of generating text.
+    class Typesafe < Provider
+      protocol :system_one, SystemOne
+
+      def api_base
+        @config.typesafe_api_base || 'https://api.typesafe.ai'
+      end
+
+      def headers
+        { 'Authorization' => "Bearer #{@config.typesafe_api_key}" }
+      end
+
+      def evaluate(state, questions, model:) # :nodoc:
+        resolve_protocol(nil, model).new(self, model).evaluate(state, questions, model: model_id_for(model))
+      end
+
+      def parse_error(response) # :nodoc:
+        body = parse_error_body(response)
+        detail = body['detail'] if body.is_a?(Hash)
+
+        case detail
+        when Hash then detail['message'] || super
+        when Array then detail.map { |part| detail_message(part) }.join('. ')
+        when String then detail
+        else super
+        end
+      end
+
+      class << self
+        def display_name
+          'TypeSafe'
+        end
+
+        def configuration_options
+          %i[typesafe_api_key typesafe_api_base]
+        end
+
+        def configuration_requirements
+          %i[typesafe_api_key]
+        end
+
+        # The model listing returns aliases only, while versioned ids such
+        # as jev-1.13.0 are accepted too.
+        def assume_models_exist?
+          true
+        end
+      end
+
+      private
+
+      def detail_message(part)
+        part.is_a?(Hash) ? part['msg'].to_s : part.to_s
+      end
+    end
+  end
+end
+
+RubyLLM::Provider.register :typesafe, RubyLLM::Providers::Typesafe,
+                           models: File.expand_path('../../../models.json', __dir__)

data/lib/ruby_llm/typesafe/evaluation.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Typesafe
+    # The answers to one Typesafe.evaluate call, keyed by the ids you gave
+    # the questions.
+    #
+    #   evaluation[:urgent].probability # => 0.92
+    #   evaluation.cost.total           # => 0.0000131
+    #
+    class Evaluation
+      include Support::Inspectable
+
+      # The answer to a yes/no question. +probability+ is the chance the
+      # answer is yes, from 0 to 1.
+      Noul = Struct.new(:probability, keyword_init: true) do
+        # Returns whether +probability+ reaches +threshold+. Tune the
+        # threshold on your own data and consequences.
+        def yes?(threshold: 0.5)
+          probability >= threshold
+        end
+      end
+
+      # The answer to a choice question. +option+ is the most probable
+      # option, +probabilities+ maps every option to its probability, and
+      # +confidence+ (0 to 1) says how concentrated they are.
+      Choice = Struct.new(:option, :probabilities, :confidence, keyword_init: true)
+
+      # The answer to a score question. +score+ is the probability-weighted
+      # level index and can land between levels. +probabilities+ holds one
+      # probability per level, in the order of +levels+.
+      Score = Struct.new(:score, :levels, :probabilities, :confidence, keyword_init: true) do
+        # Returns the level description nearest to +score+.
+        def level
+          levels[score.round]
+        end
+      end
+
+      # The answers as a Hash of question id to Noul, Choice, or Score.
+      attr_reader :answers
+
+      # The versioned id of the model that answered, such as "jev-1.13.0".
+      attr_reader :model
+
+      # The raw provider response body.
+      attr_reader :raw
+
+      # The Tokens the evaluation used.
+      attr_reader :tokens
+
+      def initialize(answers:, model:, tokens:, model_info: nil, raw: nil) # :nodoc:
+        @answers = answers
+        @model = model
+        @tokens = tokens
+        @model_info = model_info
+        @raw = raw
+      end
+
+      # Returns the answer to the question with +id+.
+      def [](id)
+        answers.fetch(id)
+      end
+
+      # Returns the evaluation's Cost, or a Cost with a +nil+ total when the
+      # model has no known pricing.
+      def cost
+        Cost.new(tokens: tokens, model: @model_info)
+      end
+
+      def inspect_attributes # :nodoc:
+        { model: model, answers: answers.keys }
+      end
+    end
+  end
+end