typesafe-ai-ruby 0.1.0 → 0.2.0

data/lib/typesafe/questions/box.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module TypeSafe
+  module Questions
+    # Locates what its instructions describe in an image. The answer is a bounding box.
+    # Box questions need an image in the request, the :images danger zone, and Jev 1.14
+    # (model "jev-preview" or "jev-1.14.0"). Box takes no criteria.
+    #
+    # For example:
+    #   TypeSafe::Box.new(instructions: "the coffee mug in `photo`")
+    class Box < Question
+      TYPE = "box"
+
+      def initialize(instructions: nil)
+        super(TYPE, instructions: instructions, criteria: nil)
+      end
+    end
+  end
+end

data/lib/typesafe/questions/choice.rb

@@ -14,10 +14,13 @@ module TypeSafe
       TYPE = "choice"
 
       def initialize(criteria:, instructions: nil)
-        super(TYPE, instructions: instructions, criteria: self.class.normalize_criteria(criteria))
+        assets = []
+        instructions = Util.deep_stringify(instructions, assets: assets) # collect instruction images first
+        super(TYPE, instructions: instructions, criteria: self.class.normalize_criteria(criteria, assets: assets),
+                    assets: assets)
       end
 
-      def self.normalize_criteria(criteria)
+      def self.normalize_criteria(criteria, assets: nil)
         raise ValidationError, "Choice criteria must be a non-empty Hash of option => description" unless
           criteria.is_a?(Hash) && !criteria.empty?
 
@@ -25,7 +28,7 @@ module TypeSafe
           name = Util.stringify_key(key)
           raise ValidationError, "Choice option names must not be blank" if name.strip.empty?
 
-          result[name] = value.nil? ? nil : Util.deep_stringify(value)
+          result[name] = value.nil? ? nil : Util.deep_stringify(value, assets: assets)
         end
       end
     end

data/lib/typesafe/questions/normalizer.rb

@@ -6,36 +6,39 @@ module TypeSafe
     # mistakes before a request is sent.
     module Normalizer
       TYPES = {
-        Noul::TYPE => Noul, Choice::TYPE => Choice, Score::TYPE => Score
+        Noul::TYPE => Noul, Choice::TYPE => Choice, Score::TYPE => Score, Box::TYPE => Box
       }.freeze
 
       module_function
 
-      def state(state)
+      # Images found in the state are appended to +assets+ and replaced by their references.
+      def state(state, assets: nil)
         raise ValidationError, "state must not be nil; pass a String, Hash, or Array" if state.nil?
 
-        value = Util.deep_stringify(state)
+        value = Util.deep_stringify(state, assets: assets)
         return value if value.is_a?(String) || value.is_a?(Hash) || value.is_a?(Array)
 
         raise ValidationError, "state must be a String, Hash, or Array, got #{state.class}"
       end
 
       # Questions in wire format.
-      def questions(questions)
+      def questions(questions, assets: nil)
         raise ValidationError, "questions must be a non-empty Hash" unless questions.is_a?(Hash) && !questions.empty?
 
         questions.each_with_object({}) do |(name, question), result|
           key = Util.stringify_key(name)
           raise ValidationError, "question names must not be blank" if key.strip.empty?
 
-          result[key] = question(question, key)
+          result[key] = question(question, key, assets: assets)
         end
       end
 
-      def question(question, name)
+      def question(question, name, assets: nil)
         case question
-        when Question then question.to_h
-        when Hash then raw_question(question, name)
+        when Question
+          assets&.concat(question.assets)
+          question.to_h
+        when Hash then raw_question(question, name, assets: assets)
         else
           raise ValidationError,
                 "question #{name.inspect} must be a TypeSafe::Noul, Choice, Score, or a Hash with a \"type\" key"
@@ -44,8 +47,8 @@ module TypeSafe
 
       # Validate a raw Hash question. Known types get the same checks as the typed classes;
       # unknown types and extra keys pass through untouched so new API fields work immediately.
-      def raw_question(hash, name)
-        wire = Util.deep_stringify(hash)
+      def raw_question(hash, name, assets: nil)
+        wire = Util.deep_stringify(hash, assets: assets)
         type = wire["type"]
         unless type.is_a?(String) && !type.empty?
           raise ValidationError,
@@ -53,7 +56,11 @@ module TypeSafe
         end
 
         klass = TYPES[type]
-        wire["criteria"] = klass.normalize_criteria(wire["criteria"]) if klass && requires_criteria?(klass, wire)
+        if klass == Box
+          raise ValidationError, "question #{name.inspect}: box questions take no criteria" if wire.key?("criteria")
+        elsif klass && requires_criteria?(klass, wire)
+          wire["criteria"] = klass.normalize_criteria(wire["criteria"])
+        end
         wire
       end
 

data/lib/typesafe/questions/noul.rb

@@ -14,10 +14,13 @@ module TypeSafe
       KEYS = %w[true false].freeze
 
       def initialize(instructions: nil, criteria: nil)
-        super(TYPE, instructions: instructions, criteria: self.class.normalize_criteria(criteria))
+        assets = []
+        instructions = Util.deep_stringify(instructions, assets: assets) # collect instruction images first
+        super(TYPE, instructions: instructions, criteria: self.class.normalize_criteria(criteria, assets: assets),
+                    assets: assets)
       end
 
-      def self.normalize_criteria(criteria)
+      def self.normalize_criteria(criteria, assets: nil)
         return nil if criteria.nil?
         raise ValidationError, "Noul criteria must be a Hash with true/false keys" unless criteria.is_a?(Hash)
 
@@ -28,7 +31,7 @@ module TypeSafe
                   "Noul criteria keys must be true or false, got #{key.inspect}"
           end
 
-          result[name] = value.nil? ? nil : Util.deep_stringify(value)
+          result[name] = value.nil? ? nil : Util.deep_stringify(value, assets: assets)
         end
       end
     end

data/lib/typesafe/questions/question.rb

@@ -11,11 +11,16 @@ module TypeSafe
       attr_reader :instructions
       # Type-specific rubric.
       attr_reader :criteria
+      # Images placed in the instructions or criteria, sent as assets with the request.
+      attr_reader :assets
 
-      def initialize(type, instructions:, criteria:)
+      def initialize(type, instructions:, criteria:, assets: [])
         @type = type
-        @instructions = instructions.nil? ? nil : Util.deep_stringify(instructions)
+        @assets = assets
+        @instructions = instructions.nil? ? nil : Util.deep_stringify(instructions, assets: @assets)
         @criteria = criteria
+        @assets.uniq!
+        @assets.freeze
         freeze
       end
 

data/lib/typesafe/questions/score.rb

@@ -14,17 +14,20 @@ module TypeSafe
       MIN_LEVELS = 2
 
       def initialize(criteria:, instructions: nil)
-        super(TYPE, instructions: instructions, criteria: self.class.normalize_criteria(criteria))
+        assets = []
+        instructions = Util.deep_stringify(instructions, assets: assets) # collect instruction images first
+        super(TYPE, instructions: instructions, criteria: self.class.normalize_criteria(criteria, assets: assets),
+                    assets: assets)
       end
 
-      def self.normalize_criteria(criteria)
+      def self.normalize_criteria(criteria, assets: nil)
         raise ValidationError, "Score criteria must be an Array of at least #{MIN_LEVELS} levels" unless
           criteria.is_a?(Array) && criteria.length >= MIN_LEVELS
 
         criteria.map do |level|
           raise ValidationError, "Score levels must not be nil" if level.nil?
 
-          Util.deep_stringify(level)
+          Util.deep_stringify(level, assets: assets)
         end
       end
     end

data/lib/typesafe/request_options.rb

@@ -3,7 +3,7 @@
 module TypeSafe
   # Per-call overrides accepted by Client#system_one and Resources::Models#list.
   class RequestOptions
-    KEYS = %i[timeout headers extra_body retry_policy].freeze
+    KEYS = %i[timeout headers extra_body retry_policy danger_zone].freeze
 
     # Timeout per attempt in seconds for this call only.
     attr_reader :timeout
@@ -13,21 +13,32 @@ module TypeSafe
     attr_reader :extra_body
     # Retry policy override for this call only: a RetryPolicy, or a Hash applied on the client policy.
     attr_reader :retry_policy
+    # Experimental features to turn on for this call: :images and/or :long_context.
+    attr_reader :danger_zone
 
-    def self.from(options)
-      case options
-      when RequestOptions then options
-      when nil then new
-      when Hash then new(**options.transform_keys(&:to_sym))
-      else raise ArgumentError, "request_options must be a Hash, got #{options.class}"
-      end
+    # Coerce nil, a Hash, or a RequestOptions. A danger_zone given separately is merged in.
+    def self.from(options, danger_zone: nil)
+      result = case options
+               when RequestOptions then options
+               when nil then new
+               when Hash then new(**options.transform_keys(&:to_sym))
+               else raise ArgumentError, "request_options must be a Hash, got #{options.class}"
+               end
+      danger_zone.nil? ? result : result.with_danger_zone(danger_zone)
     end
 
-    def initialize(timeout: nil, headers: nil, extra_body: nil, retry_policy: nil)
+    # A copy with additional danger zones turned on.
+    def with_danger_zone(zones)
+      self.class.new(timeout: timeout, headers: headers, extra_body: extra_body, retry_policy: retry_policy,
+                     danger_zone: danger_zone + Util.danger_zones(zones))
+    end
+
+    def initialize(timeout: nil, headers: nil, extra_body: nil, retry_policy: nil, danger_zone: nil)
       @timeout = validate_timeout(timeout)
       @headers = (headers || {}).each_with_object({}) { |(name, value), result| result[name.to_s] = value.to_s }
       @extra_body = validate_extra_body(extra_body)
       @retry_policy = validate_retry_policy(retry_policy)
+      @danger_zone = Util.danger_zones(danger_zone).freeze
       freeze
     end
 

data/lib/typesafe/responses/answer.rb

@@ -129,6 +129,79 @@ module TypeSafe
       end
     end
 
+    # The four edges of a box as fractions of the image's width (left, right) and height
+    # (top, bottom), measured from the top-left corner.
+    class BoxCoordinates
+      attr_reader :top, :bottom, :left, :right
+
+      def self.parse(data, path)
+        box = Reader.hash!(data, path)
+        new(**%i[top bottom left right].to_h { |edge| [edge, Reader.number!(box[edge.to_s], "#{path}.#{edge}")] })
+      end
+
+      def initialize(top:, bottom:, left:, right:)
+        @top = top
+        @bottom = bottom
+        @left = left
+        @right = right
+        freeze
+      end
+
+      def width
+        right - left
+      end
+
+      def height
+        bottom - top
+      end
+
+      # [left, top, right, bottom], the order most image libraries use for a crop box.
+      def to_a
+        [left, top, right, bottom]
+      end
+
+      # [left, top, right, bottom] in integer pixels for an image of the given size.
+      def to_pixels(width:, height:)
+        [(left * width).round, (top * height).round, (right * width).round, (bottom * height).round]
+      end
+
+      def to_h
+        { "top" => top, "bottom" => bottom, "left" => left, "right" => right }
+      end
+
+      def ==(other)
+        other.is_a?(BoxCoordinates) && to_a == other.to_a
+      end
+      alias eql? ==
+
+      def hash
+        to_a.hash
+      end
+    end
+
+    # A located box with its confidence.
+    class BoxAnswer < Answer
+      attr_reader :box
+      # Confidence from 0 to 1, or nil when the API did not report it.
+      attr_reader :confidence
+
+      def self.parse(data, path)
+        confidence = data["confidence"]
+        new(
+          data,
+          box: BoxCoordinates.parse(data["box"], "#{path}.box"),
+          confidence: confidence.nil? ? nil : Reader.number!(confidence, "#{path}.confidence")
+        )
+      end
+
+      def initialize(raw, box:, confidence:)
+        @box = box
+        @confidence = confidence
+        super(raw)
+        freeze
+      end
+    end
+
     # An answer whose type this gem does not recognize. Only #raw is available.
     class UnknownAnswer < Answer
       def self.parse(data, _path)
@@ -137,7 +210,7 @@ module TypeSafe
     end
 
     Answer::TYPES = {
-      "noul" => NoulAnswer, "choice" => ChoiceAnswer, "score" => ScoreAnswer
+      "noul" => NoulAnswer, "choice" => ChoiceAnswer, "score" => ScoreAnswer, "box" => BoxAnswer
     }.freeze
   end
 end

data/lib/typesafe/responses/system_one_response.rb

@@ -71,6 +71,10 @@ module TypeSafe
         answers_of(ScoreAnswer)
       end
 
+      def boxes
+        answers_of(BoxAnswer)
+      end
+
       def to_h
         raw || { "model" => model, "usage" => usage.to_h, "answers" => answers.transform_values(&:to_h) }
       end

data/lib/typesafe/util.rb

@@ -30,21 +30,30 @@ module TypeSafe
     end
 
     # Convert a value into plain JSON-compatible Ruby: string keys, no symbols, finite floats.
-    # Objects responding to as_json are converted through it. Anything else raises.
-    def deep_stringify(value)
+    # Objects responding to as_json are converted through it. Images become their reference and
+    # are appended to +assets+; without an +assets+ collector an Image raises. Anything else raises.
+    def deep_stringify(value, assets: nil)
       case value
       when String, Integer, true, false, nil then value
       when Symbol then value.to_s
       when Float then finite_float(value)
-      when Hash, Array then stringify_container(value)
-      else convert_object(value)
+      when Image then image_reference(value, assets)
+      when Hash, Array then stringify_container(value, assets)
+      else convert_object(value, assets)
       end
     end
 
-    def stringify_container(value)
-      return value.map { |v| deep_stringify(v) } if value.is_a?(Array)
+    def image_reference(image, assets)
+      raise ValidationError, "images can only be placed in state, instructions and criteria" if assets.nil?
 
-      value.each_with_object({}) { |(k, v), h| h[stringify_key(k)] = deep_stringify(v) }
+      assets << image
+      image.reference
+    end
+
+    def stringify_container(value, assets)
+      return value.map { |v| deep_stringify(v, assets: assets) } if value.is_a?(Array)
+
+      value.each_with_object({}) { |(k, v), h| h[stringify_key(k)] = deep_stringify(v, assets: assets) }
     end
 
     def finite_float(value)
@@ -61,7 +70,7 @@ module TypeSafe
       end
     end
 
-    def convert_object(value)
+    def convert_object(value, assets)
       unless value.respond_to?(:as_json)
         raise ValidationError,
               "cannot encode #{value.class} as JSON; pass a String, Hash, Array, or an object responding to as_json"
@@ -70,7 +79,7 @@ module TypeSafe
       converted = value.as_json
       raise ValidationError, "#{value.class}#as_json returned itself and cannot be encoded" if converted.equal?(value)
 
-      deep_stringify(converted)
+      deep_stringify(converted, assets: assets)
     end
 
     # Parse retry-after-ms (preferred) or Retry-After (seconds or HTTP-date) into seconds.
@@ -112,6 +121,17 @@ module TypeSafe
       JSON.generate(body)
     end
 
+    # Normalize a danger_zone option into an Array of known Symbols.
+    def danger_zones(value)
+      zones = Array(value).map { |zone| zone.to_s.to_sym }.uniq
+      unknown = zones - Constants::DANGER_ZONES.keys
+      return zones if unknown.empty?
+
+      raise ArgumentError,
+            "unknown danger_zone #{unknown.map(&:inspect).join(", ")}; " \
+            "expected #{Constants::DANGER_ZONES.keys.map(&:inspect).join(", ")}"
+    end
+
     # The runtime identifier sent in X-TypeSafe-Runtime.
     def runtime
       "ruby/#{RUBY_VERSION} (#{RUBY_PLATFORM})"

data/lib/typesafe/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module TypeSafe
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/typesafe.rb

@@ -14,14 +14,18 @@ require_relative "typesafe/logging"
 require_relative "typesafe/instrumentation"
 require_relative "typesafe/configuration"
 require_relative "typesafe/retry_policy"
+require_relative "typesafe/image"
+require_relative "typesafe/assets"
 require_relative "typesafe/request_options"
 require_relative "typesafe/questions/question"
 require_relative "typesafe/questions/noul"
 require_relative "typesafe/questions/choice"
 require_relative "typesafe/questions/score"
+require_relative "typesafe/questions/box"
 require_relative "typesafe/questions/normalizer"
 require_relative "typesafe/http/request"
 require_relative "typesafe/http/response"
+require_relative "typesafe/http/multipart_encoder"
 require_relative "typesafe/http/connection_manager"
 require_relative "typesafe/http/net_http_transport"
 require_relative "typesafe/http/retrier"
@@ -41,6 +45,7 @@ module TypeSafe
   Noul = Questions::Noul
   Choice = Questions::Choice
   Score = Questions::Score
+  Box = Questions::Box
 
   class << self
     # Global settings used by .client. Mutate through .configure.
@@ -102,6 +107,14 @@ module TypeSafe
       Score.new(instructions: instructions, criteria: criteria)
     end
 
+    # Build a box question.
+    #
+    # For example:
+    #   TypeSafe.box("the coffee mug in `photo`")
+    def box(instructions = nil)
+      Box.new(instructions: instructions)
+    end
+
     private
 
     def merge_criteria(positional, named)

data/sig/typesafe/http.rbs

@@ -11,8 +11,10 @@ module TypeSafe
       attr_reader headers: Hash[String, String]
       attr_reader body: String?
       attr_reader timeout: Numeric
+      attr_reader body_summary: String?
 
-      def initialize: (method: Symbol, uri: URI::HTTP, headers: Hash[String, String], body: String?, timeout: Numeric) -> void
+      def initialize: (method: Symbol, uri: URI::HTTP, headers: Hash[String, String], body: String?, timeout: Numeric,
+                       ?body_summary: String?) -> void
       def endpoint: () -> String
       def with_headers: (Hash[String, String] extra) -> Request
     end
@@ -31,6 +33,29 @@ module TypeSafe
       def retry_after: () -> Float?
     end
 
+    class MultipartEncoder
+      class Part
+        attr_accessor name: String
+        attr_accessor content_type: String
+        attr_accessor content: String
+
+        def initialize: (name: String, content_type: String, content: String) -> void
+      end
+
+      class Encoded
+        attr_accessor body: String
+        attr_accessor content_type: String
+        attr_accessor summary: String
+      end
+
+      attr_reader boundary: String
+
+      def self.build: (Hash[String, untyped] body, Hash[String, Image] assets) -> Encoded
+      def initialize: (?boundary: String) -> void
+      def content_type: () -> String
+      def encode: (String json, Array[Part] parts) -> String
+    end
+
     class ConnectionManager
       IDLE_TIMEOUT: Integer
 
@@ -69,7 +94,7 @@ module TypeSafe
 
       def initialize: (Configuration config, ?transport: _Transport?, ?sleeper: (^(Numeric) -> void)?,
                        ?clock: (^() -> Float)?) -> void
-      def post: (String path, body: Hash[String, untyped], ?options: RequestOptions) -> Response
+      def post: (String path, body: Hash[String, untyped], ?options: RequestOptions, ?assets: Hash[String, Image]) -> Response
       def get: (String path, ?options: RequestOptions) -> Response
       def close: () -> void
     end

data/sig/typesafe/questions.rbs

@@ -4,8 +4,9 @@ module TypeSafe
       attr_reader type: String
       attr_reader instructions: json_content?
       attr_reader criteria: untyped
+      attr_reader assets: Array[Image]
 
-      def initialize: (String type, instructions: json_input?, criteria: untyped) -> void
+      def initialize: (String type, instructions: json_input?, criteria: untyped, ?assets: Array[Image]) -> void
       def to_h: () -> Hash[String, untyped]
       def to_json: (*untyped args) -> String
       def ==: (untyped other) -> bool
@@ -18,14 +19,14 @@ module TypeSafe
       KEYS: Array[String]
 
       def initialize: (?instructions: json_input?, ?criteria: Hash[untyped, untyped]?) -> void
-      def self.normalize_criteria: (Hash[untyped, untyped]? criteria) -> Hash[String, json_content?]?
+      def self.normalize_criteria: (Hash[untyped, untyped]? criteria, ?assets: Array[Image]?) -> Hash[String, json_content?]?
     end
 
     class Choice < Question
       TYPE: String
 
       def initialize: (?instructions: json_input?, criteria: Hash[untyped, untyped]) -> void
-      def self.normalize_criteria: (Hash[untyped, untyped] criteria) -> Hash[String, json_content?]
+      def self.normalize_criteria: (Hash[untyped, untyped] criteria, ?assets: Array[Image]?) -> Hash[String, json_content?]
     end
 
     class Score < Question
@@ -33,16 +34,22 @@ module TypeSafe
       MIN_LEVELS: Integer
 
       def initialize: (?instructions: json_input?, criteria: Array[untyped]) -> void
-      def self.normalize_criteria: (Array[untyped] criteria) -> Array[json_content]
+      def self.normalize_criteria: (Array[untyped] criteria, ?assets: Array[Image]?) -> Array[json_content]
+    end
+
+    class Box < Question
+      TYPE: String
+
+      def initialize: (?instructions: json_input?) -> void
     end
 
     module Normalizer
       TYPES: Hash[String, singleton(Question)]
 
-      def self?.state: (json_input state) -> json_content
-      def self?.questions: (Hash[String | Symbol, question_input] questions) -> Hash[String, Hash[String, untyped]]
-      def self?.question: (question_input question, String name) -> Hash[String, untyped]
-      def self?.raw_question: (Hash[untyped, untyped] hash, String name) -> Hash[String, untyped]
+      def self?.state: (json_input | Image state, ?assets: Array[Image]?) -> json_content
+      def self?.questions: (Hash[String | Symbol, question_input] questions, ?assets: Array[Image]?) -> Hash[String, Hash[String, untyped]]
+      def self?.question: (question_input question, String name, ?assets: Array[Image]?) -> Hash[String, untyped]
+      def self?.raw_question: (Hash[untyped, untyped] hash, String name, ?assets: Array[Image]?) -> Hash[String, untyped]
     end
   end
 end

data/sig/typesafe/responses.rbs

@@ -73,6 +73,32 @@ module TypeSafe
       def raw_probabilities: () -> Hash[String, Numeric]
     end
 
+    class BoxCoordinates
+      attr_reader top: Float
+      attr_reader bottom: Float
+      attr_reader left: Float
+      attr_reader right: Float
+
+      def self.parse: (untyped data, String path) -> BoxCoordinates
+      def initialize: (top: Float, bottom: Float, left: Float, right: Float) -> void
+      def width: () -> Float
+      def height: () -> Float
+      def to_a: () -> [Float, Float, Float, Float]
+      def to_pixels: (width: Numeric, height: Numeric) -> [Integer, Integer, Integer, Integer]
+      def to_h: () -> Hash[String, Float]
+      def ==: (untyped other) -> bool
+      alias eql? ==
+      def hash: () -> Integer
+    end
+
+    class BoxAnswer < Answer
+      attr_reader box: BoxCoordinates
+      attr_reader confidence: Float?
+
+      def self.parse: (Hash[String, untyped] data, String path) -> BoxAnswer
+      def initialize: (Hash[String, untyped] raw, box: BoxCoordinates, confidence: Float?) -> void
+    end
+
     class UnknownAnswer < Answer
       def self.parse: (Hash[String, untyped] data, String path) -> UnknownAnswer
     end
@@ -94,6 +120,7 @@ module TypeSafe
       def nouls: () -> IndifferentHash
       def choices: () -> IndifferentHash
       def scores: () -> IndifferentHash
+      def boxes: () -> IndifferentHash
       def to_h: () -> Hash[String, untyped]
       def to_json: (*untyped args) -> String
     end