mistri 0.0.2 → 0.1.0

data/lib/mistri/tool_result.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Mistri
+  # A two-channel tool result: content goes to the model, ui goes only to the
+  # host. The ui payload rides the tool message and its :tool_result event,
+  # persists with the session for transcript re-renders, and never reaches a
+  # provider. Return one from a handler when the UI needs more than the model
+  # should read or pay for: full query rows behind a compact answer, the
+  # updated document behind "saved".
+  #
+  #   Tool.define("edit_page", "Edits the page.") do |args|
+  #     page = apply(args)
+  #     Mistri::ToolResult.new(content: "Updated.", ui: { "html" => page })
+  #   end
+  #
+  # ui must be JSON-serializable; it is stored and delivered in canonical
+  # JSON form (string keys), the same shape a reloaded session reads.
+  ToolResult = Data.define(:content, :ui) do
+    def initialize(content:, ui: nil)
+      super
+    end
+  end
+end

data/lib/mistri/tools/edit_file.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Mistri
+  module Tools
+    module_function
+
+    # The model-facing shape is flat {path, old_string, new_string,
+    # replace_all} on purpose: it is the shape frontier models are trained on,
+    # and nested edit arrays measurably degrade their calls. Failures come
+    # back in band with the closest region and its exact difference, so the
+    # model's retry is one shot.
+    def edit_file(workspace)
+      Tool.define("edit_file",
+                  "Replace an exact snippet of a document. Copy old_string verbatim from " \
+                  "read_file output including whitespace, without line-number prefixes. " \
+                  "It must match exactly one place; add surrounding lines to make it " \
+                  "unique, or set replace_all to change every occurrence.",
+                  eager_input_streaming: true,
+                  schema: lambda {
+                    string :path, "Document path", required: true
+                    string :old_string, "Exact text to replace (whitespace matters)", required: true
+                    string :new_string, "Replacement text", required: true
+                    boolean :replace_all, "Replace every occurrence instead of exactly one"
+                  }) do |args|
+        args = Tools.tolerate(args)
+        with_document(workspace, args) do |content|
+          result = Edit.replace(content, args["old_string"], args["new_string"],
+                                replace_all: args["replace_all"] == true)
+          workspace.write(args["path"], result.content)
+          "Replaced #{result.count} occurrence(s) in #{args["path"]}"
+        end
+      rescue EditError => e
+        "edit_file failed: #{e.message}"
+      end
+    end
+  end
+end

data/lib/mistri/tools/find_in_file.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Mistri
+  module Tools
+    module_function
+
+    def find_in_file(workspace)
+      Tool.define("find_in_file",
+                  "Find text in a document. Returns line-numbered matches with context, " \
+                  "so you can locate a region without reading the whole document.",
+                  schema: lambda {
+                    string :path, "Document path", required: true
+                    string :query, "Text to find (plain substring)", required: true
+                    integer :context, "Context lines around each match"
+                  }) do |args|
+        with_document(workspace, args) do |content|
+          Tools.find_matches(content, args["query"], (args["context"] || 2).to_i)
+        end
+      end
+    end
+
+    def find_matches(content, query, context)
+      lines = content.lines
+      hits = lines.each_index.select { |i| lines[i].include?(query) }
+      return "No matches for #{query.inspect}." if hits.empty?
+
+      blocks = hits.first(20).map do |hit|
+        from = [hit - context, 0].max
+        to = [hit + context, lines.length - 1].min
+        (from..to).map { |n| "#{n + 1}: #{lines[n]}" }.join
+      end
+      notice = hits.length > 20 ? "\n[#{hits.length - 20} more matches not shown]" : ""
+      "#{blocks.join("---\n")}#{notice}"
+    end
+  end
+end

data/lib/mistri/tools/list_files.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Mistri
+  module Tools
+    module_function
+
+    def list_files(workspace)
+      Tool.define("list_files",
+                  "List document paths in the workspace, optionally under a prefix.",
+                  schema: -> { string :prefix, "Only paths starting with this" }) do |args|
+        paths = workspace.list(args["prefix"])
+        paths.empty? ? "No documents found." : paths.join("\n")
+      end
+    end
+  end
+end

data/lib/mistri/tools/read_file.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Mistri
+  module Tools
+    MAX_READ_CHARS = 20_000
+
+    module_function
+
+    def read_file(workspace)
+      Tool.define("read_file",
+                  "Read a document with line numbers. Use offset and limit for a window " \
+                  "into a long document.",
+                  schema: lambda {
+                    string :path, "Document path", required: true
+                    integer :offset, "First line to read (1-based)"
+                    integer :limit, "How many lines to read"
+                  }) do |args|
+        with_document(workspace, args) do |content|
+          Tools.numbered_window(content, args["offset"], args["limit"])
+        end
+      end
+    end
+
+    def numbered_window(content, offset, limit)
+      lines = content.lines
+      from = [(offset || 1).to_i, 1].max
+      to = limit ? [from + limit.to_i - 1, lines.length].min : lines.length
+      numbered = (from..to).map { |n| "#{n}: #{lines[n - 1]}" }.join
+      windowed = to < lines.length || from > 1
+      suffix = windowed ? "\n[showing lines #{from}-#{to} of #{lines.length}]" : ""
+      return "#{numbered}#{suffix}" if numbered.length <= MAX_READ_CHARS
+
+      cut = numbered[0, MAX_READ_CHARS]
+      cut = cut[0..(cut.rindex("\n") || -1)]
+      "#{cut}\n[truncated at #{MAX_READ_CHARS} chars; use offset/limit to read more]"
+    end
+  end
+end

data/lib/mistri/tools/read_memory.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Mistri
+  module Tools
+    module_function
+
+    def read_memory(memory)
+      Tool.define("read_memory",
+                  "Read the durable memory: knowledge kept across sessions. Check it " \
+                  "before starting work that earlier sessions may have learned about.") do |_args|
+        content = memory.read
+        content.empty? ? "Memory is empty." : content
+      end
+    end
+  end
+end

data/lib/mistri/tools/update_memory.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Mistri
+  module Tools
+    module_function
+
+    # Whole-document replace on purpose: the model rewrites memory as one
+    # coherent text instead of appending fragments that drift.
+    def update_memory(memory)
+      Tool.define("update_memory",
+                  "Replace the durable memory with an updated version. Pass the FULL " \
+                  "text: what you were given plus what you learned, rewritten to stay " \
+                  "short and current.",
+                  schema: lambda {
+                    string :content, "The complete new memory text", required: true
+                  }) do |args|
+        memory.replace(args["content"])
+        "Memory updated (#{args["content"].to_s.length} chars)."
+      end
+    end
+  end
+end

data/lib/mistri/tools/write_file.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Mistri
+  module Tools
+    module_function
+
+    def write_file(workspace)
+      Tool.define("write_file",
+                  "Create or fully overwrite a document with the given content.",
+                  eager_input_streaming: true,
+                  schema: lambda {
+                    string :path, "Document path", required: true
+                    string :content, "The full document content", required: true
+                  }) do |args|
+        workspace.write(args["path"], args["content"])
+        "Wrote #{args["path"]} (#{args["content"].to_s.length} chars)"
+      end
+    end
+  end
+end

data/lib/mistri/tools.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Mistri
+  # Built-in tools. Tools.files binds the document tools to a workspace, so
+  # "file" means whatever the workspace says it means: a database column, a
+  # row in a documents table, or an actual file. The names stay read_file and
+  # edit_file because those are the tool names models are trained on.
+  module Tools
+    ALIASES = { "oldText" => "old_string", "old" => "old_string", "search" => "old_string",
+                "newText" => "new_string", "new" => "new_string", "replace" => "new_string",
+                "replaceAll" => "replace_all", "file" => "path", "filename" => "path" }.freeze
+
+    module_function
+
+    def files(workspace)
+      [read_file(workspace), write_file(workspace), edit_file(workspace),
+       find_in_file(workspace), list_files(workspace)]
+    end
+
+    def memory(store)
+      [read_memory(store), update_memory(store)]
+    end
+
+    def with_document(workspace, args)
+      content = workspace.read(args["path"])
+      return "No document at #{args["path"].inspect}. Use list_files to see paths." if content.nil?
+
+      yield content
+    end
+
+    # Absorb the drift real models produce: alias keys, stringly booleans,
+    # unknown keys dropped by simply never being read.
+    def tolerate(args)
+      normalized = args.to_h { |key, value| [ALIASES.fetch(key.to_s, key.to_s), value] }
+      case normalized["replace_all"]
+      when "true", "1", 1 then normalized["replace_all"] = true
+      when "false", "0", 0, nil then normalized["replace_all"] = false
+      end
+      normalized
+    end
+  end
+end
+
+require_relative "tools/read_file"
+require_relative "tools/write_file"
+require_relative "tools/edit_file"
+require_relative "tools/find_in_file"
+require_relative "tools/list_files"
+require_relative "tools/read_memory"
+require_relative "tools/update_memory"

data/lib/mistri/transport.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+require "uri"
+
+module Mistri
+  # HTTP for one provider origin, held open across the turns of a run: a
+  # multi-turn agent pays the TCP and TLS handshake once, not per turn. Not
+  # shareable across threads; a mutex serializes accidental concurrent use onto
+  # the single socket, and re-entering from a streaming callback raises
+  # ThreadError.
+  #
+  # Streaming reads abort two ways: cooperatively between fragments, and hard,
+  # by closing the socket from the abort signal's callback, so a stalled read
+  # stops immediately instead of waiting out the read timeout.
+  class Transport
+    KEEP_ALIVE_SECONDS = 30
+
+    def initialize(origin:, open_timeout: 15, read_timeout: 300, write_timeout: 60)
+      @origin = origin.to_s.chomp("/")
+      @uri = URI(@origin)
+      @open_timeout = open_timeout
+      @read_timeout = read_timeout
+      @write_timeout = write_timeout
+      @mutex = Mutex.new
+      @connection = nil
+    end
+
+    # POST and decode a JSON response body. Retries once on a dead idle
+    # socket, so it suits idempotent endpoints.
+    def post(path, body:, headers: {})
+      response = @mutex.synchronize do
+        with_retry { connection.request(build_request(path, body, headers)) }
+      end
+      raise_for_status(response)
+      JSON.parse(response.body)
+    end
+
+    # POST and stream the SSE response, yielding each decoded data record.
+    # Returns :aborted when the signal cancelled the stream, else nil.
+    def stream_post(path, body:, headers: {}, signal: nil, &block)
+      return :aborted if signal&.aborted?
+
+      @mutex.synchronize { stream_locked(path, body, headers, signal, &block) }
+    end
+
+    def close
+      @mutex.synchronize { teardown }
+    end
+
+    private
+
+    def stream_locked(path, body, headers, signal, &block)
+      retried = false
+      begin
+        started = false
+        aborted = false
+        conn = connection
+        # The closer targets this turn's connection, so an abort that fires
+        # late, after the turn already finished, cannot touch a successor's
+        # socket.
+        closer = signal&.on_abort { quietly_finish(conn) }
+        begin
+          conn.request(build_request(path, body, headers, streaming: true)) do |response|
+            started = true
+            raise_for_status(response)
+            aborted = read_stream(response, signal, &block)
+          end
+        ensure
+          signal&.remove_callback(closer) if closer
+        end
+        # An abort mid-body leaves unread bytes on the socket; drop it rather
+        # than let the next request read stale frames.
+        teardown if aborted
+        aborted ? :aborted : nil
+      rescue IOError, SocketError, SystemCallError, Timeout::Error => e
+        teardown
+        return :aborted if signal&.aborted?
+
+        # A dead idle keep-alive socket fails before the response starts and
+        # retries safely. A drop after events flowed must not replay the turn.
+        if !started && !retried
+          retried = true
+          retry
+        end
+        raise ProviderError, "connection lost mid-stream: #{e.message}"
+      end
+    end
+
+    def read_stream(response, signal, &block)
+      sse = SSE.new
+      aborted = false
+      response.read_body do |fragment|
+        if signal&.aborted?
+          aborted = true
+          break
+        end
+        sse.feed(fragment, &block)
+      end
+      sse.finish(&block) unless aborted
+      aborted
+    end
+
+    def build_request(path, body, headers, streaming: false)
+      request = Net::HTTP::Post.new(URI("#{@origin}#{path}"))
+      request["Content-Type"] = "application/json"
+      if streaming
+        request["Accept"] = "text/event-stream"
+        # Net::HTTP silently negotiates gzip, and its inflater buffers the
+        # whole stream, delivering "live" events in one burst at the end.
+        request["Accept-Encoding"] = "identity"
+      end
+      headers.each { |key, value| request[key] = value }
+      request.body = JSON.generate(body)
+      request
+    end
+
+    def with_retry
+      attempted = false
+      begin
+        yield
+      rescue Timeout::Error => e
+        # The server may already be executing the request; a replay risks
+        # running it twice.
+        teardown
+        raise ProviderError, "request timed out: #{e.message}"
+      rescue IOError, SocketError, SystemCallError => e
+        teardown
+        raise ProviderError, "connection failed: #{e.message}" if attempted
+
+        attempted = true
+        retry
+      end
+    end
+
+    def connection
+      @connection ||= Net::HTTP.new(@uri.host, @uri.port).tap do |http|
+        http.use_ssl = @uri.scheme == "https"
+        http.open_timeout = @open_timeout
+        http.read_timeout = @read_timeout
+        http.write_timeout = @write_timeout
+        http.keep_alive_timeout = KEEP_ALIVE_SECONDS
+        http.start
+      end
+    end
+
+    def teardown
+      @connection&.finish
+    rescue IOError
+      nil
+    ensure
+      @connection = nil
+    end
+
+    def quietly_finish(conn)
+      conn.finish
+    rescue IOError
+      nil
+    end
+
+    def raise_for_status(response)
+      status = response.code.to_i
+      return if (200..299).cover?(status)
+
+      klass = error_class(status)
+      options = { status: status, body: response.read_body.to_s[0, 500] }
+      options[:retry_after] = retry_after(response) if klass == RateLimitError
+      raise klass.new(**options)
+    end
+
+    def error_class(status)
+      case status
+      when 401, 403 then AuthenticationError
+      when 429 then RateLimitError
+      when 529 then OverloadedError
+      when 500..599 then ServerError
+      else ProviderError
+      end
+    end
+
+    def retry_after(response)
+      value = response["retry-after"]
+      value&.match?(/\A\d+(\.\d+)?\z/) ? value.to_f : nil
+    end
+  end
+end

data/lib/mistri/usage.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Mistri
+  # Token accounting for an assistant turn, and the dollar cost of those tokens.
+  #
+  # `input` counts only prompt tokens billed at the full rate: cache reads and
+  # writes are separate fields, so a cached token is never double-billed.
+  # `reasoning` is the thinking slice of `output`. `cache_write_1h` is the slice
+  # of `cache_write` held for an hour, which bills at twice the input rate.
+  class Usage < Data.define(:input, :output, :cache_read, :cache_write,
+                            :cache_write_1h, :reasoning, :cost)
+    Cost = Data.define(:input, :output, :cache_read, :cache_write, :total) do
+      def self.zero = new(input: 0.0, output: 0.0, cache_read: 0.0, cache_write: 0.0, total: 0.0)
+
+      def +(other)
+        self.class.new(input: input + other.input, output: output + other.output,
+                       cache_read: cache_read + other.cache_read,
+                       cache_write: cache_write + other.cache_write,
+                       total: total + other.total)
+      end
+    end
+
+    def initialize(input: 0, output: 0, cache_read: 0, cache_write: 0,
+                   cache_write_1h: 0, reasoning: 0, cost: Cost.zero)
+      super
+    end
+
+    def self.zero = new
+
+    def self.from_h(hash)
+      h = (hash || {}).transform_keys(&:to_s)
+      c = (h["cost"] || {}).transform_keys(&:to_s)
+      new(input: h.fetch("input", 0).to_i, output: h.fetch("output", 0).to_i,
+          cache_read: h.fetch("cache_read", 0).to_i, cache_write: h.fetch("cache_write", 0).to_i,
+          cache_write_1h: h.fetch("cache_write_1h", 0).to_i,
+          reasoning: h.fetch("reasoning", 0).to_i,
+          cost: Cost.new(input: c.fetch("input", 0).to_f, output: c.fetch("output", 0).to_f,
+                         cache_read: c.fetch("cache_read", 0).to_f,
+                         cache_write: c.fetch("cache_write", 0).to_f,
+                         total: c.fetch("total", 0).to_f))
+    end
+
+    def total_tokens = input + output + cache_read + cache_write
+
+    # A copy with cost computed from per-million-token rates. The 1h cache-write
+    # slice bills at the :cache_write_1h rate when given, else at twice the
+    # input rate, matching extended-retention pricing.
+    def with_cost(rates)
+      long = [cache_write_1h, cache_write].min
+      short = cache_write - long
+      computed = {
+        input: rate(rates, :input) * input,
+        output: rate(rates, :output) * output,
+        cache_read: rate(rates, :cache_read) * cache_read,
+        cache_write: (rate(rates, :cache_write) * short) + (long_write_rate(rates) * long)
+      }
+      with(cost: Cost.new(**computed, total: computed.values.sum))
+    end
+
+    def +(other)
+      self.class.new(input: input + other.input, output: output + other.output,
+                     cache_read: cache_read + other.cache_read,
+                     cache_write: cache_write + other.cache_write,
+                     cache_write_1h: cache_write_1h + other.cache_write_1h,
+                     reasoning: reasoning + other.reasoning,
+                     cost: cost + other.cost)
+    end
+
+    def to_h = super.merge(cost: cost.to_h)
+
+    private
+
+    def rate(rates, key) = rates.fetch(key, 0).to_f / 1_000_000
+
+    def long_write_rate(rates)
+      rates.key?(:cache_write_1h) ? rate(rates, :cache_write_1h) : rate(rates, :input) * 2
+    end
+  end
+end

data/lib/mistri/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Mistri
-  VERSION = "0.0.2".freeze
+  VERSION = "0.1.0"
 end

data/lib/mistri/workspace/active_record.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Mistri
+  module Workspace
+    # Documents in the host's own database, through a model class the host
+    # supplies, optionally scoped (per session, per tenant). Not auto-required;
+    # load it with require "mistri/workspace/active_record".
+    #
+    # The model needs a path column and a content column, unique on path
+    # within the scope. A migration to copy:
+    #
+    #   create_table :mistri_documents do |t|
+    #     t.string :session_id, null: false
+    #     t.string :path, null: false, limit: 512
+    #     t.text :content, size: :medium, null: false
+    #     t.timestamps
+    #   end
+    #   add_index :mistri_documents, [:session_id, :path], unique: true
+    class ActiveRecord
+      def initialize(model, scope: {})
+        @model = model
+        @scope = scope
+      end
+
+      def read(path)
+        @model.where(**@scope, path: path.to_s).pick(:content)
+      end
+
+      def write(path, content)
+        record = @model.find_or_initialize_by(**@scope, path: path.to_s)
+        record.content = content.to_s
+        record.save!
+        nil
+      end
+
+      def delete(path)
+        @model.where(**@scope, path: path.to_s).delete_all
+        nil
+      end
+
+      def list(prefix = nil)
+        paths = @model.where(**@scope).pluck(:path).sort
+        prefix ? paths.select { |p| p.start_with?(prefix.to_s) } : paths
+      end
+    end
+  end
+end

data/lib/mistri/workspace/directory.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Mistri
+  module Workspace
+    # Documents as files under one root. Every path resolves inside the root
+    # or raises, so a model-supplied "../../etc/passwd" cannot escape.
+    class Directory
+      def initialize(root)
+        @root = File.expand_path(root)
+        FileUtils.mkdir_p(@root)
+      end
+
+      def read(path)
+        full = resolve(path)
+        File.exist?(full) ? File.read(full) : nil
+      end
+
+      def write(path, content)
+        full = resolve(path)
+        FileUtils.mkdir_p(File.dirname(full))
+        File.write(full, content.to_s)
+        nil
+      end
+
+      def delete(path)
+        full = resolve(path)
+        FileUtils.rm_f(full)
+        nil
+      end
+
+      def list(prefix = nil)
+        base = @root.length + 1
+        paths = Dir.glob(File.join(@root, "**", "*")).select { |f| File.file?(f) }
+                                                     .map { |f| f[base..] }.sort
+        prefix ? paths.select { |p| p.start_with?(prefix.to_s) } : paths
+      end
+
+      private
+
+      def resolve(path)
+        full = File.expand_path(path.to_s, @root)
+        unless full == @root || full.start_with?("#{@root}#{File::SEPARATOR}")
+          raise SchemaError, "path escapes the workspace: #{path}"
+        end
+
+        full
+      end
+    end
+  end
+end

data/lib/mistri/workspace/memory.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Mistri
+  # The document store agents work in. A workspace maps paths to text and can
+  # live anywhere: memory for tests and ephemeral runs, a directory when disk
+  # exists, the host's database when it does not. The file tools bind to this
+  # port, so fuzzy-editing a MySQL row works exactly like editing a file.
+  #
+  # A backend implements read(path) -> String or nil, write(path, content),
+  # delete(path), and list(prefix = nil) -> [paths].
+  module Workspace
+    class Memory
+      def initialize
+        @documents = {}
+        @mutex = Mutex.new
+      end
+
+      def read(path)
+        @mutex.synchronize { @documents[path.to_s] }
+      end
+
+      def write(path, content)
+        @mutex.synchronize { @documents[path.to_s] = content.to_s.dup.freeze }
+        nil
+      end
+
+      def delete(path)
+        @mutex.synchronize { @documents.delete(path.to_s) }
+        nil
+      end
+
+      def list(prefix = nil)
+        @mutex.synchronize do
+          keys = @documents.keys.sort
+          prefix ? keys.select { |key| key.start_with?(prefix.to_s) } : keys
+        end
+      end
+    end
+  end
+end