longctx-bench 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 0f98993121e27fa709664d7599ea7a7cbf0028c4c6d5d9c5c276b2a937cfce4a
+  data.tar.gz: 87f7967b4b12c7f0af6ff631e73d4184f13aaeae280263bde86c3e5d463a976e
+SHA512:
+  metadata.gz: d322ef706527530288850d3b69a17fec5711e9a34cb26ce3bd6780a453385e961aa80f506c464cc5d667efd067890e854412af7ab543bb6dd606a55ae135650f
+  data.tar.gz: 03f3b353bef3b0be7dce6c838a3d9f7c975c3a7f1e5ddb03281cd9b0fd896e21ceae41fd57cff8461ce4045d58449abe6b2dbcf0ea51b3fd5009d0b8a43ae05e

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Alex Rivera
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,44 @@
+# longctx-bench
+
+A dependency-free Ruby harness for measuring how OpenAI-compatible chat
+endpoints behave under long-context continuation loads.
+
+It builds deterministic prompts of a target approximate token budget
+(~4 chars/token), sends them as a single user message, and records the HTTP
+status, latency, token usage and the first 200 characters of the answer.
+
+## Install
+
+    gem install longctx-bench
+
+## Usage
+
+    export LONGCTX_BASE_URL=https://api.example.com
+    export LONGCTX_API_KEY=sk-...
+    longctx-bench --model example-model --tokens 100000 --runs 1 --out run.json
+
+Twenty long runs (the usual "does the free tier survive" check):
+
+    longctx-bench --model example-model --tokens 200000 --runs 20 --out soak.json
+
+## Output
+
+One line per run plus an optional JSON file:
+
+    run=1 est_tokens=100012 status=200 latency_ms=8421 usage={"prompt_tokens"=>100012,...} error=nil
+
+Status `402`/`429` in the JSON file means the provider ran out of credits or
+rate limit during the soak, which is exactly what this tool is meant to
+surface.
+
+## Library use
+
+```ruby
+require "longctx/bench"
+Longctx.run(base_url: "https://api.example.com", api_key: ENV["KEY"],
+            model: "example-model", tokens: 200_000, runs: 20)
+```
+
+## License
+
+MIT

data/exe/longctx-bench

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "optparse"
+require "longctx/bench"
+
+opts = {
+  base_url: ENV["LONGCTX_BASE_URL"],
+  api_key: ENV["LONGCTX_API_KEY"],
+  model: ENV["LONGCTX_MODEL"],
+  tokens: 100_000,
+  runs: 1,
+  seed: 42,
+  out: nil
+}
+
+OptionParser.new do |o|
+  o.banner = "Usage: longctx-bench --base-url URL --model NAME [options]"
+  o.on("--base-url URL", "OpenAI-compatible base URL") { |v| opts[:base_url] = v }
+  o.on("--api-key KEY", "Bearer token (or LONGCTX_API_KEY)") { |v| opts[:api_key] = v }
+  o.on("--model NAME", "Model id to call") { |v| opts[:model] = v }
+  o.on("--tokens N", Integer, "Approximate input tokens per run (default 100000)") { |v| opts[:tokens] = v }
+  o.on("--runs N", Integer, "Number of runs (default 1)") { |v| opts[:runs] = v }
+  o.on("--seed N", Integer, "Prompt seed (default 42)") { |v| opts[:seed] = v }
+  o.on("--out FILE", "Write JSON results to FILE") { |v| opts[:out] = v }
+end.parse!
+
+abort "missing --base-url" unless opts[:base_url]
+abort "missing --model" unless opts[:model]
+
+Longctx.run(**opts)

data/lib/longctx/bench.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative "version"
+require_relative "prompt"
+require_relative "client"
+
+module Longctx
+  # Tiny benchmark harness for long-context continuation workloads.
+  #
+  #   Longctx.run(base_url: "https://api.example.com", api_key: ENV["KEY"],
+  #               model: "example-model", tokens: 100_000, runs: 1)
+  module_function
+
+  def run(base_url:, api_key:, model:, tokens: 100_000, runs: 1, seed: 42, out: nil)
+    client = Client.new(base_url: base_url, api_key: api_key, model: model)
+    results = (1..runs).map do |i|
+      prompt, est = Prompt.build(target_tokens: tokens, seed: seed, chapter: i)
+      res = client.complete(prompt, run: i)
+      line = format("run=%d est_tokens=%d status=%d latency_ms=%d usage=%s error=%s",
+                    i, est, res.status, res.latency_ms, res.usage.inspect, res.error.inspect)
+      puts line
+      { run: i, est_tokens: est, status: res.status, latency_ms: res.latency_ms,
+        usage: res.usage, error: res.error, preview: res.preview }
+    end
+    File.write(out, JSON.pretty_generate(results)) if out
+    results
+  end
+end
+

data/lib/longctx/client.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "uri"
+
+module Longctx
+  # Minimal OpenAI-compatible chat-completions client used by the benchmark.
+  # Deliberately dependency-free so the gem can run anywhere Ruby runs.
+  class Client
+    Result = Struct.new(:run, :status, :latency_ms, :usage, :error, :preview,
+                        keyword_init: true)
+
+    def initialize(base_url:, api_key:, model:, timeout: 600)
+      @base_url = base_url.sub(%r{/+\z}, "")
+      @api_key = api_key
+      @model = model
+      @timeout = timeout
+    end
+
+    def complete(prompt, run: 1, max_tokens: 400)
+      uri = URI.parse("#{@base_url}/v1/chat/completions")
+      req = Net::HTTP::Post.new(uri)
+      req["Content-Type"] = "application/json"
+      req["Authorization"] = "Bearer #{@api_key}" if @api_key && !@api_key.empty?
+      req.body = JSON.generate(
+        model: @model,
+        messages: [{ role: "user", content: prompt }],
+        max_tokens: max_tokens,
+        temperature: 0.7
+      )
+
+      started = Time.now
+      begin
+        res = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
+                              read_timeout: @timeout, open_timeout: 30) do |http|
+          http.request(req)
+        end
+        latency = ((Time.now - started) * 1000).round
+        payload = JSON.parse(res.body) rescue {}
+        usage = payload["usage"]
+        preview = payload.dig("choices", 0, "message", "content").to_s[0, 200]
+        error = nil
+        if res.code.to_i >= 400
+          error = payload["error"].is_a?(Hash) ? payload["error"]["message"] : res.body.to_s[0, 200]
+        end
+        Result.new(run: run, status: res.code.to_i, latency_ms: latency, usage: usage,
+                   error: error, preview: preview)
+      rescue StandardError => e
+        latency = ((Time.now - started) * 1000).round
+        Result.new(run: run, status: 0, latency_ms: latency, usage: nil,
+                   error: "#{e.class}: #{e.message}", preview: "")
+      end
+    end
+  end
+end

data/lib/longctx/prompt.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Longctx
+  # Builds deterministic, reproducible prompts of a target approximate
+  # token budget. Token counts are estimated with the usual ~4 chars/token
+  # heuristic; the estimate is printed alongside every run so results stay
+  # comparable across providers.
+  module Prompt
+    CHARS_PER_TOKEN = 4
+
+    SENTENCES = [
+      "The harbour lights blinked twice before the fog swallowed them whole.",
+      "She counted the bells of the old clock tower and lost track at eleven.",
+      "Every map in the archive agreed on one thing: the river had moved.",
+      "He wrote the same paragraph four times and each version ended differently.",
+      "The train was late, the platform empty, the timetable a rumour.",
+      "Rain drummed on the tin roof like a drummer who had forgotten the song.",
+      "They kept the letters in a biscuit tin under the stairs.",
+      "The lighthouse keeper had not spoken to anyone in nine days.",
+      "A paper boat drifted past the window and nobody mentioned it.",
+      "In the morning the snow had erased the road and the argument alike."
+    ].freeze
+
+    module_function
+
+    # Returns [text, estimated_tokens]
+    def build(target_tokens:, seed: 42, chapter: 1)
+      rng = Random.new(seed + chapter)
+      target_chars = [target_tokens * CHARS_PER_TOKEN, 200].max
+      body = +"Chapter #{chapter}. "
+      body << "Context bundle for a long-form continuation experiment.\n\n"
+      while body.length < target_chars
+        body << SENTENCES[rng.rand(SENTENCES.length)] << " "
+        body << "(note #{rng.rand(100_000)})\n" if rng.rand(25).zero?
+      end
+      body << "\n\nContinue the story for roughly 300 words. Keep the same narrator, " \
+              "reuse at most two named characters, and end on an unresolved image."
+      [body, (body.length.to_f / CHARS_PER_TOKEN).round]
+    end
+  end
+end

data/lib/longctx/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Longctx
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,56 @@
+--- !ruby/object:Gem::Specification
+name: longctx-bench
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Alex Rivera
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2026-10-22 00:00:00.000000000 Z
+dependencies: []
+description: Builds deterministic ~100k/200k token prompts, calls an OpenAI-compatible
+  /v1/chat/completions endpoint and records status, latency, usage and errors. Intended
+  for evaluating free-tier limits under long-context novel-continuation workloads.
+email:
+- longctxbench@users.noreply.github.com
+executables:
+- longctx-bench
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- exe/longctx-bench
+- lib/longctx/bench.rb
+- lib/longctx/client.rb
+- lib/longctx/prompt.rb
+- lib/longctx/version.rb
+homepage: https://rubygems.org/gems/longctx-bench
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://rubygems.org/gems/longctx-bench
+  documentation_uri: https://rubygems.org/gems/longctx-bench
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.15
+signing_key:
+specification_version: 4
+summary: Dependency-free long-context continuation benchmark for OpenAI-compatible
+  endpoints
+test_files: []