dexter_llm 0.1.2

data/lib/dexter_llm/content.rb

@@ -0,0 +1,338 @@
+# frozen_string_literal: true
+
+module DexterLlm
+  module Content
+    module Serializable
+      def to_h
+        raise NotImplementedError
+      end
+    end
+
+    class Text
+      include Serializable
+
+      def initialize(text, text_signature: nil)
+        @text = text.to_s
+        @text_signature = text_signature
+      end
+
+      attr_reader :text, :text_signature
+      def type = :text
+      def to_h = { "type" => "text", "text" => text, "text_signature" => text_signature }.compact
+    end
+
+    class Image
+      include Serializable
+
+      def initialize(data:, mime_type:)
+        @data = data
+        @mime_type = mime_type
+      end
+
+      attr_reader :data, :mime_type
+      def type = :image
+      def to_h = { "type" => "image", "data" => data, "mime_type" => mime_type }
+    end
+
+    module DocumentSource
+      class Store
+        include Serializable
+
+        def initialize(document_id)
+          @document_id = document_id
+        end
+
+        attr_reader :document_id
+        def type = :store
+        def to_h = { "type" => "store", "document_id" => document_id }
+      end
+
+      class Url
+        include Serializable
+
+        def initialize(url)
+          @url = url
+        end
+
+        attr_reader :url
+        def type = :url
+        def to_h = { "type" => "url", "url" => url }
+      end
+
+      class Base64
+        include Serializable
+
+        def initialize(data:, mime_type:)
+          @data = data
+          @mime_type = mime_type
+        end
+
+        attr_reader :data, :mime_type
+        def type = :base64
+        def to_h = { "type" => "base64", "data" => data, "mime_type" => mime_type }
+      end
+
+      class ProviderFile
+        include Serializable
+
+        def initialize(provider:, file_id:)
+          @provider = provider
+          @file_id = file_id
+        end
+
+        attr_reader :provider, :file_id
+        def type = :provider_file
+        def to_h = { "type" => "provider_file", "provider" => provider.to_s, "file_id" => file_id }
+      end
+    end
+
+    class Document
+      include Serializable
+
+      def initialize(
+        source:,
+        filename:,
+        mime_type:,
+        byte_size: nil,
+        sha256: nil,
+        title: nil,
+        context: nil,
+        citations_enabled: nil,
+        label: nil
+      )
+        @source = source
+        @filename = filename
+        @mime_type = mime_type
+        @byte_size = byte_size
+        @sha256 = sha256
+        @title = title
+        @context = context
+        @citations_enabled = citations_enabled
+        @label = label
+      end
+
+      attr_reader :source, :filename, :mime_type, :byte_size, :sha256,
+        :title, :context, :citations_enabled, :label
+
+      def type = :document
+
+      def to_h
+        {
+          "type" => "document",
+          "source" => source.respond_to?(:to_h) ? source.to_h : source,
+          "filename" => filename,
+          "mime_type" => mime_type,
+          "byte_size" => byte_size,
+          "sha256" => sha256,
+          "title" => title,
+          "context" => context,
+          "citations_enabled" => citations_enabled,
+          "label" => label
+        }.compact
+      end
+    end
+
+    class ToolCall
+      include Serializable
+
+      def initialize(id:, name:, arguments:, thought_signature: nil)
+        @id = id
+        @name = name
+        @arguments = arguments || {}
+        @thought_signature = thought_signature
+      end
+
+      attr_reader :id, :name, :arguments, :thought_signature
+      def type = :tool_call
+      def to_h
+        {
+          "type" => "tool_call",
+          "id" => id,
+          "name" => name,
+          "arguments" => arguments,
+          "thought_signature" => thought_signature
+        }.compact
+      end
+    end
+
+    class Thinking
+      include Serializable
+
+      def initialize(thinking, thinking_signature: nil)
+        @thinking = thinking.to_s
+        @thinking_signature = thinking_signature
+      end
+
+      attr_reader :thinking, :thinking_signature
+      def type = :thinking
+      def to_h = { "type" => "thinking", "thinking" => thinking, "thinking_signature" => thinking_signature }.compact
+    end
+
+    # Represents a server-side tool invocation (built-in tools like web_search, web_fetch).
+    # Unlike ToolCall which represents user-defined tools, ServerToolUse represents
+    # tools executed by the provider on the server side.
+    class ServerToolUse
+      include Serializable
+
+      def initialize(id:, name:, input:)
+        @id = id
+        @name = name
+        @input = input || {}
+      end
+
+      attr_reader :id, :name, :input
+      def type = :server_tool_use
+
+      def to_h
+        {
+          "type" => "server_tool_use",
+          "id" => id,
+          "name" => name,
+          "input" => input
+        }
+      end
+    end
+
+    # Represents an individual web search result.
+    class WebSearchResult
+      include Serializable
+
+      def initialize(url:, title:, encrypted_content: nil, page_age: nil)
+        @url = url
+        @title = title
+        @encrypted_content = encrypted_content
+        @page_age = page_age
+      end
+
+      attr_reader :url, :title, :encrypted_content, :page_age
+      def type = :web_search_result
+
+      def to_h
+        {
+          "type" => "web_search_result",
+          "url" => url,
+          "title" => title,
+          "encrypted_content" => encrypted_content,
+          "page_age" => page_age
+        }.compact
+      end
+    end
+
+    # Represents the result of a web search tool invocation.
+    class WebSearchToolResult
+      include Serializable
+
+      def initialize(tool_use_id:, results: [], error: nil)
+        @tool_use_id = tool_use_id
+        @results = results
+        @error = error
+      end
+
+      attr_reader :tool_use_id, :results, :error
+      def type = :web_search_tool_result
+
+      def error?
+        !@error.nil?
+      end
+
+      def to_h
+        hash = {
+          "type" => "web_search_tool_result",
+          "tool_use_id" => tool_use_id
+        }
+        if error?
+          hash["error"] = error
+        else
+          hash["results"] = results.map { |r| r.respond_to?(:to_h) ? r.to_h : r }
+        end
+        hash
+      end
+    end
+
+    # Represents the result of a web fetch tool invocation.
+    class WebFetchToolResult
+      include Serializable
+
+      def initialize(tool_use_id:, url: nil, content: nil, title: nil, retrieved_at: nil, error: nil)
+        @tool_use_id = tool_use_id
+        @url = url
+        @content = content
+        @title = title
+        @retrieved_at = retrieved_at
+        @error = error
+      end
+
+      attr_reader :tool_use_id, :url, :content, :title, :retrieved_at, :error
+      def type = :web_fetch_tool_result
+
+      def error?
+        !@error.nil?
+      end
+
+      def to_h
+        hash = {
+          "type" => "web_fetch_tool_result",
+          "tool_use_id" => tool_use_id
+        }
+        if error?
+          hash["error"] = error
+        else
+          hash["url"] = url
+          hash["content"] = content
+          hash["title"] = title if title
+          hash["retrieved_at"] = retrieved_at if retrieved_at
+        end
+        hash.compact
+      end
+    end
+
+    # Represents a citation from web search or web fetch results.
+    class Citation
+      include Serializable
+
+      def initialize(type:, url: nil, title: nil, cited_text: nil, start_index: nil, end_index: nil, encrypted_index: nil)
+        @citation_type = type
+        @url = url
+        @title = title
+        @cited_text = cited_text
+        @start_index = start_index
+        @end_index = end_index
+        @encrypted_index = encrypted_index
+      end
+
+      attr_reader :citation_type, :url, :title, :cited_text, :start_index, :end_index, :encrypted_index
+      def type = :citation
+
+      def to_h
+        {
+          "type" => citation_type,
+          "url" => url,
+          "title" => title,
+          "cited_text" => cited_text,
+          "start_index" => start_index,
+          "end_index" => end_index,
+          "encrypted_index" => encrypted_index
+        }.compact
+      end
+    end
+
+    # Text content with optional citations (used in web search responses).
+    class CitedText
+      include Serializable
+
+      def initialize(text, citations: [])
+        @text = text.to_s
+        @citations = citations
+      end
+
+      attr_reader :text, :citations
+      def type = :cited_text
+
+      def to_h
+        hash = { "type" => "cited_text", "text" => text }
+        hash["citations"] = citations.map { |c| c.respond_to?(:to_h) ? c.to_h : c } if citations.any?
+        hash
+      end
+    end
+  end
+end

data/lib/dexter_llm/context_overflow_error.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DexterLlm
+  class ContextOverflowError < ApiError; end
+end

data/lib/dexter_llm/documents/ingestor.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "digest"
+require "marcel"
+require "stringio"
+
+module DexterLlm
+  module Documents
+    class Ingestor
+      MAX_FILE_SIZE = 100 * 1024 * 1024 # 100MB
+
+      ALLOWED_MIME_TYPES = %w[
+        application/pdf
+        text/plain
+        text/csv
+      ].freeze
+
+      class << self
+        # Ingest file and store
+        # @param io [IO, String] File handle or bytes
+        # @param filename [String] Original filename
+        # @param mime_type [String, nil] Explicit MIME type (auto-detected if nil)
+        # @param store [Store] Document store
+        # @return [StoredDocument] The stored document
+        # @raise [InvalidRequestError] if validation fails
+        def ingest(io, filename:, store:, mime_type: nil)
+          io = normalize_io(io)
+
+          # 1. Detect MIME type if not provided
+          detected_mime = mime_type || detect_mime_type(io, filename)
+          io.rewind
+
+          # 2. Validate MIME type
+          validate_mime_type!(detected_mime)
+
+          # 3. Compute sha256 + validate size
+          sha256, byte_size = compute_digest_and_size(io)
+          validate_size!(byte_size)
+          io.rewind
+
+          # 4. Store via Store#put
+          document_id = store.put(io, filename: filename, mime_type: detected_mime, sha256: sha256)
+
+          # 5. Return StoredDocument
+          StoredDocument.new(
+            document_id: document_id,
+            filename: filename,
+            mime_type: detected_mime,
+            byte_size: byte_size,
+            sha256: sha256
+          )
+        end
+
+        private
+
+        def normalize_io(io)
+          case io
+          when String
+            StringIO.new(io)
+          when IO, StringIO
+            io
+          else
+            if io.respond_to?(:read)
+              io
+            else
+              raise InvalidRequestError.new("Invalid input: expected IO, String, or readable object", status: 400)
+            end
+          end
+        end
+
+        def detect_mime_type(io, filename)
+          Marcel::MimeType.for(io, name: filename)
+        end
+
+        def validate_mime_type!(mime_type)
+          return if ALLOWED_MIME_TYPES.include?(mime_type)
+
+          raise InvalidRequestError.new(
+            "Unsupported file type: #{mime_type}. Allowed types: #{ALLOWED_MIME_TYPES.join(', ')}",
+            status: 400
+          )
+        end
+
+        def validate_size!(byte_size)
+          return if byte_size <= MAX_FILE_SIZE
+
+          raise InvalidRequestError.new(
+            "File too large: #{byte_size} bytes (max #{MAX_FILE_SIZE / 1024 / 1024}MB)",
+            status: 400
+          )
+        end
+
+        def compute_digest_and_size(io)
+          digest = Digest::SHA256.new
+          size = 0
+
+          while (chunk = io.read(16384))
+            digest.update(chunk)
+            size += chunk.bytesize
+          end
+
+          [ digest.hexdigest, size ]
+        end
+      end
+    end
+  end
+end

data/lib/dexter_llm/documents/store.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module DexterLlm
+  module Documents
+    class Store
+      # Store document bytes, return sha256 identifier
+      # @param io [IO, String] File handle or bytes
+      # @param filename [String] Original filename
+      # @param mime_type [String] MIME type
+      # @param sha256 [String, nil] Pre-computed sha256 (computed if nil)
+      # @return [String] sha256 hex digest (document_id)
+      def put(io, filename:, mime_type:, sha256: nil)
+        raise NotImplementedError
+      end
+
+      # Retrieve document metadata
+      # @param document_id [String] sha256 hex digest
+      # @return [StoredDocument, nil] Document object or nil if not found
+      def get(document_id)
+        raise NotImplementedError
+      end
+
+      # Open document bytes as IO stream
+      # @param document_id [String] sha256 hex digest
+      # @yield [IO] Block receives IO object
+      # @return [Object] Block result
+      def open(document_id, &block)
+        raise NotImplementedError
+      end
+
+      # Check if document exists
+      # @param document_id [String] sha256 hex digest
+      # @return [Boolean]
+      def exists?(document_id)
+        raise NotImplementedError
+      end
+
+      # Delete a document
+      # @param document_id [String] sha256 hex digest
+      # @return [Boolean] true if deleted, false if not found
+      def delete(document_id)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/dexter_llm/documents/stored_document.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module DexterLlm
+  module Documents
+    class StoredDocument
+      attr_reader :document_id, :filename, :mime_type, :byte_size, :sha256
+
+      def initialize(document_id:, filename:, mime_type:, byte_size:, sha256: nil)
+        @document_id = document_id
+        @filename = filename
+        @mime_type = mime_type
+        @byte_size = byte_size
+        @sha256 = sha256 || document_id
+      end
+
+      def to_h
+        {
+          "document_id" => document_id,
+          "filename" => filename,
+          "mime_type" => mime_type,
+          "byte_size" => byte_size,
+          "sha256" => sha256
+        }
+      end
+    end
+  end
+end

data/lib/dexter_llm/documents/stores/file_system.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "pathname"
+require "digest"
+
+module DexterLlm
+  module Documents
+    module Stores
+      class FileSystem < Store
+        def initialize(root: nil)
+          @root = Pathname.new(root || default_root)
+        end
+
+        def put(io, filename:, mime_type:, sha256: nil)
+          # Compute sha256 if not provided
+          if sha256.nil?
+            sha256, _size = compute_digest(io)
+            io.rewind
+          end
+
+          # Content-addressable: use sha256 as path
+          ensure_directory!
+          content_path = path_for(sha256)
+          meta_path = meta_path_for(sha256)
+
+          # Write content file using exclusive create (atomic deduplication)
+          begin
+            File.open(content_path, File::WRONLY | File::CREAT | File::EXCL, 0644) do |f|
+              while (chunk = io.read(16384))
+                f.write(chunk)
+              end
+            end
+          rescue Errno::EEXIST
+            # File already exists - deduplication in action, continue to update metadata
+          end
+
+          # Always update metadata (filename/mime_type might differ)
+          byte_size = content_path.size
+          metadata = {
+            "filename" => filename,
+            "mime_type" => mime_type,
+            "byte_size" => byte_size,
+            "sha256" => sha256
+          }
+          File.write(meta_path, JSON.generate(metadata))
+
+          sha256
+        end
+
+        def get(document_id)
+          meta_path = meta_path_for(document_id)
+          return nil unless meta_path.exist?
+
+          metadata = JSON.parse(File.read(meta_path))
+          StoredDocument.new(
+            document_id: document_id,
+            filename: metadata["filename"],
+            mime_type: metadata["mime_type"],
+            byte_size: metadata["byte_size"],
+            sha256: metadata["sha256"] || document_id
+          )
+        end
+
+        def open(document_id, &block)
+          content_path = path_for(document_id)
+          raise InvalidRequestError.new("Document not found: #{document_id}", status: 404) unless content_path.exist?
+
+          File.open(content_path, "rb", &block)
+        end
+
+        def exists?(document_id)
+          path_for(document_id).exist?
+        end
+
+        def delete(document_id)
+          content_path = path_for(document_id)
+          meta_path = meta_path_for(document_id)
+
+          return false unless content_path.exist?
+
+          FileUtils.rm_f(content_path)
+          FileUtils.rm_f(meta_path)
+          true
+        end
+
+        private
+
+        attr_reader :root
+
+        def default_root
+          if defined?(Rails)
+            Rails.root.join("storage", "documents").to_s
+          else
+            File.join(Dir.pwd, "storage", "documents")
+          end
+        end
+
+        def ensure_directory!
+          FileUtils.mkdir_p(root)
+        end
+
+        def path_for(document_id)
+          root.join(sanitize_id(document_id))
+        end
+
+        def meta_path_for(document_id)
+          root.join("#{sanitize_id(document_id)}.meta.json")
+        end
+
+        def sanitize_id(document_id)
+          # sha256 hex is 64 chars of [0-9a-f], but validate anyway
+          document_id.to_s.gsub(/[^a-f0-9]/, "")
+        end
+
+        def compute_digest(io)
+          digest = Digest::SHA256.new
+          size = 0
+
+          while (chunk = io.read(16384))
+            digest.update(chunk)
+            size += chunk.bytesize
+          end
+
+          [ digest.hexdigest, size ]
+        end
+      end
+    end
+  end
+end

data/lib/dexter_llm/error.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DexterLlm
+  class Error < StandardError; end
+end

data/lib/dexter_llm/instrumentation.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module DexterLlm
+  module Instrumentation
+    class Null
+      def instrument(_name, **_payload)
+        yield if block_given?
+      end
+    end
+  end
+end

data/lib/dexter_llm/invalid_request_error.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DexterLlm
+  class InvalidRequestError < ApiError; end
+end

data/lib/dexter_llm/message.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module DexterLlm
+  module Message
+    module Serializable
+      def to_h
+        raise NotImplementedError
+      end
+
+      private
+
+      def normalize_content(content)
+        case content
+        when String
+          [ Content::Text.new(content) ]
+        when Array
+          content
+        when nil
+          []
+        else
+          [ Content::Text.new(content.to_s) ]
+        end
+      end
+
+      def content_to_h(content)
+        Array(content).map { |c| c.respond_to?(:to_h) ? c.to_h : c }
+      end
+    end
+  end
+end