raimei-bench 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +13 -0
- data/README.md +3 -0
- data/exe/raimei-bench +62 -0
- data/lib/raimei/bench/report.rb +55 -0
- data/lib/raimei/bench/version.rb +1 -1
- data/lib/raimei/bench.rb +99 -10
- metadata +10 -5
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 15fa8bff21ad42eef9d31f70b09725e018fa800f78b1bf9e9328fceec1cce417
|
|
4
|
+
data.tar.gz: f3e50c0d28eb25cb04cb0dc5744276923fece6ca56255396bb5c2fe00baca8ee
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: bdf9771fb6a5fbdfe2741fef1026e1a509b543fbb0f7a1b45e2530e0e8f9a0bec355137789f034c77c265ac515fee3dd39cb26feec16c63b54026f047bc20857
|
|
7
|
+
data.tar.gz: 9fd10c15d9060703f16390cfb364e991c13520011cb6fcb9d7e2ee05052410c9f4d95a8d3746848c26efaf86e07dfa1a80e7a7d8542b99c0bd7bdc5514479919
|
data/Gemfile
ADDED
data/README.md
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
# Raimei::Bench
|
|
2
2
|
|
|
3
|
+
> ⚠️ Experimental: APIs may change.
|
|
4
|
+
|
|
5
|
+
|
|
3
6
|
TODO: Delete this and the text below, and describe your gem
|
|
4
7
|
|
|
5
8
|
Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/raimei/bench`. To experiment with that code, run `bin/console` for an interactive prompt.
|
data/exe/raimei-bench
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
# Add local libs so this works without Bundler
|
|
5
|
+
root = File.expand_path("../../..", __dir__)
|
|
6
|
+
%w[raimei-bench raimei-nim raimei-shared].each do |g|
|
|
7
|
+
$LOAD_PATH.unshift File.join(root, "gems", g, "lib")
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
require "optparse"
|
|
11
|
+
require "yaml"
|
|
12
|
+
require "raimei/bench"
|
|
13
|
+
require "raimei/bench/report"
|
|
14
|
+
|
|
15
|
+
def env_or_literal(field)
|
|
16
|
+
case field
|
|
17
|
+
when String
|
|
18
|
+
if (m = field.match(/\AENV\((.+)\)\z/))
|
|
19
|
+
ENV[m[1]]
|
|
20
|
+
else
|
|
21
|
+
field
|
|
22
|
+
end
|
|
23
|
+
else
|
|
24
|
+
nil
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
cmd = ARGV.shift or abort "Usage: raimei-bench <run|report> ..."
|
|
29
|
+
|
|
30
|
+
case cmd
|
|
31
|
+
when "run"
|
|
32
|
+
opts = { url: ENV["URL"], model: ENV["MODEL"], api_key: ENV["API_KEY"], stream: true }
|
|
33
|
+
OptionParser.new do |o|
|
|
34
|
+
o.banner = "Usage: raimei-bench run <scenario.yml> [--url URL] [--model MODEL] [--api-key KEY] [--no-stream]"
|
|
35
|
+
o.on("--url URL"){|v| opts[:url]=v}
|
|
36
|
+
o.on("--model MODEL"){|v| opts[:model]=v}
|
|
37
|
+
o.on("--api-key KEY"){|v| opts[:api_key]=v}
|
|
38
|
+
o.on("--no-stream"){ opts[:stream]=false }
|
|
39
|
+
end.parse!
|
|
40
|
+
file = ARGV.shift or abort "scenario.yml required"
|
|
41
|
+
y = YAML.load_file(file)
|
|
42
|
+
name = y["name"] || File.basename(file, ".*")
|
|
43
|
+
url = opts[:url] || env_or_literal(y["url"]) || ENV["URL"]
|
|
44
|
+
model = opts[:model] || env_or_literal(y["model"]) || ENV["MODEL"]
|
|
45
|
+
stream = opts.key?(:stream) ? opts[:stream] : (y["stream"] != false)
|
|
46
|
+
prompts = y["prompts"]
|
|
47
|
+
prompts = File.expand_path(prompts, File.dirname(file)) if prompts.is_a?(String)
|
|
48
|
+
|
|
49
|
+
abort "URL missing (set --url or URL env)" unless url && !url.empty?
|
|
50
|
+
abort "MODEL missing (set --model or MODEL)" unless model && !model.empty?
|
|
51
|
+
|
|
52
|
+
sc = Raimei::Bench::Scenario.new(name: name, url: url, model: model, api_key: opts[:api_key], stream: stream)
|
|
53
|
+
sc.run(prompts)
|
|
54
|
+
|
|
55
|
+
when "report"
|
|
56
|
+
path = ARGV.shift or abort "Usage: raimei-bench report reports/<name>.json"
|
|
57
|
+
out = Raimei::Bench::Report.write(path)
|
|
58
|
+
puts "wrote #{out}"
|
|
59
|
+
|
|
60
|
+
else
|
|
61
|
+
abort "Unknown command: #{cmd}"
|
|
62
|
+
end
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require "json"
|
|
3
|
+
require "erb"
|
|
4
|
+
|
|
5
|
+
module Raimei
|
|
6
|
+
module Bench
|
|
7
|
+
module Report
|
|
8
|
+
TEMPLATE = <<~HTML
|
|
9
|
+
<!doctype html><meta charset="utf-8">
|
|
10
|
+
<title><%= name %> – Raimei Bench</title>
|
|
11
|
+
<style>
|
|
12
|
+
body{font-family:system-ui,-apple-system,Segoe UI,Roboto,Ubuntu,"Helvetica Neue",sans-serif;max-width:920px;margin:32px auto;padding:0 12px}
|
|
13
|
+
table{border-collapse:collapse;width:100%} th,td{border:1px solid #ddd;padding:8px} th{background:#f7f7f7;text-align:left}
|
|
14
|
+
.sum{margin:12px 0 20px}
|
|
15
|
+
.muted{color:#666}
|
|
16
|
+
</style>
|
|
17
|
+
<h1><%= name %></h1>
|
|
18
|
+
<p class="sum">Count: <b><%= data.length %></b> • p50: <b><%= p50 %> ms</b> • p95: <b><%= p95 %> ms</b><% if ttfb_p50 %> • TTFB p50: <b><%= ttfb_p50 %> ms</b><% end %></p>
|
|
19
|
+
<table>
|
|
20
|
+
<tr><th>#</th><th>Latency (ms)</th><th>TTFB (ms)</th><th>Prompt</th><th>Output len</th></tr>
|
|
21
|
+
<% data.each_with_index do |r,i| t=r[:telemetry]; %>
|
|
22
|
+
<tr>
|
|
23
|
+
<td><%= i+1 %></td>
|
|
24
|
+
<td><%= t[:latency_ms] %></td>
|
|
25
|
+
<td><%= t.dig(:extra,:ttfb_ms) || "" %></td>
|
|
26
|
+
<td class="muted"><%= r[:prompt] %></td>
|
|
27
|
+
<td><%= t.dig(:extra,:output_len) %></td>
|
|
28
|
+
</tr>
|
|
29
|
+
<% end %>
|
|
30
|
+
</table>
|
|
31
|
+
HTML
|
|
32
|
+
|
|
33
|
+
def self.write(json_path, html_path=nil)
|
|
34
|
+
data = JSON.parse(File.read(json_path), symbolize_names: true)
|
|
35
|
+
times = data.map { |r| r[:telemetry][:latency_ms] }
|
|
36
|
+
ttfb = data.map { |r| r[:telemetry].dig(:extra, :ttfb_ms) }.compact
|
|
37
|
+
p50 = pctile(times, 50)
|
|
38
|
+
p95 = pctile(times, 95)
|
|
39
|
+
ttfb_p50 = ttfb.empty? ? nil : pctile(ttfb, 50)
|
|
40
|
+
name = File.basename(json_path, ".json")
|
|
41
|
+
html = ERB.new(TEMPLATE).result(binding)
|
|
42
|
+
out = html_path || json_path.sub(/\.json\z/, ".html")
|
|
43
|
+
File.write(out, html)
|
|
44
|
+
out
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def self.pctile(arr, p)
|
|
48
|
+
return nil if arr.empty?
|
|
49
|
+
a = arr.sort; k=(p/100.0)*(a.length-1); f=k.floor; c=k.ceil
|
|
50
|
+
return a[f] if f==c
|
|
51
|
+
a[f] + (a[c]-a[f])*(k-f)
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
data/lib/raimei/bench/version.rb
CHANGED
data/lib/raimei/bench.rb
CHANGED
|
@@ -1,29 +1,118 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "json"
|
|
4
|
+
require "yaml"
|
|
4
5
|
require "raimei/shared"
|
|
5
6
|
require "raimei/nim"
|
|
6
7
|
|
|
7
8
|
module Raimei
|
|
8
9
|
module Bench
|
|
9
10
|
class Scenario
|
|
10
|
-
|
|
11
|
-
|
|
11
|
+
# name: report base name (also printed in summary)
|
|
12
|
+
# url: OpenAI-compatible /v1/chat/completions endpoint
|
|
13
|
+
# model: model id/name for the provider
|
|
14
|
+
# api_key: optional bearer token
|
|
15
|
+
# stream: true -> measure time-to-first-byte (TTFB) + total; false -> total only
|
|
16
|
+
# outdir: directory for report artifacts (JSON now; HTML can be added by CLI)
|
|
17
|
+
def initialize(name:, url:, model:, api_key: nil, stream: true, outdir: "reports")
|
|
18
|
+
@name, @url, @model, @api_key, @stream, @outdir =
|
|
19
|
+
name, url, model, api_key, stream, outdir
|
|
12
20
|
end
|
|
13
21
|
|
|
14
|
-
|
|
15
|
-
|
|
22
|
+
# prompts_or_path:
|
|
23
|
+
# * Array<String> -> prompts directly
|
|
24
|
+
# * "path/to/file.jsonl" -> lines like {"prompt":"..."}
|
|
25
|
+
# * "path/to/file.yml|.yaml" -> must contain key "prompts" pointing to file or array
|
|
26
|
+
# * "path/to/file.txt" -> one prompt per line
|
|
27
|
+
#
|
|
28
|
+
# Returns the in-memory results array; also writes JSON to @outdir/<name>.json
|
|
29
|
+
def run(prompts_or_path)
|
|
30
|
+
prompts = load_prompts(prompts_or_path)
|
|
31
|
+
client = Raimei::NIM::Client.new(url: @url, api_key: @api_key)
|
|
32
|
+
|
|
16
33
|
results = []
|
|
17
|
-
|
|
34
|
+
latencies = []
|
|
35
|
+
ttfbs = []
|
|
36
|
+
|
|
37
|
+
prompts.each do |prompt|
|
|
18
38
|
t0 = Raimei::Shared.now_ms
|
|
19
|
-
out =
|
|
20
|
-
|
|
21
|
-
|
|
39
|
+
out = +""
|
|
40
|
+
ttfb_ms = nil
|
|
41
|
+
|
|
42
|
+
if @stream
|
|
43
|
+
client.chat(model: @model, messages: [{ role: "user", content: prompt }], stream: true).each do |delta|
|
|
44
|
+
ttfb_ms ||= (Raimei::Shared.now_ms - t0)
|
|
45
|
+
out << delta
|
|
46
|
+
end
|
|
47
|
+
else
|
|
48
|
+
out = client.chat(model: @model, messages: [{ role: "user", content: prompt }], stream: false)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
ev = Raimei::Shared.finish(
|
|
52
|
+
name: @name, start_ms: t0, model: @model,
|
|
53
|
+
tokens_in: nil, tokens_out: nil,
|
|
54
|
+
extra: { prompt_len: prompt.size, output_len: out.size, ttfb_ms: ttfb_ms }
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
latencies << ev.latency_ms
|
|
58
|
+
ttfbs << ttfb_ms if ttfb_ms
|
|
59
|
+
results << { prompt: prompt, output: out, telemetry: ev.to_h }
|
|
22
60
|
end
|
|
23
|
-
|
|
24
|
-
|
|
61
|
+
|
|
62
|
+
Dir.mkdir(@outdir) unless Dir.exist?(@outdir)
|
|
63
|
+
json_path = File.join(@outdir, "#{@name}.json")
|
|
64
|
+
File.write(json_path, JSON.pretty_generate(results))
|
|
65
|
+
|
|
66
|
+
summary = {
|
|
67
|
+
count: latencies.size,
|
|
68
|
+
p50_ms: pctile(latencies, 50),
|
|
69
|
+
p95_ms: pctile(latencies, 95),
|
|
70
|
+
ttfb_p50_ms: ttfbs.empty? ? nil : pctile(ttfbs, 50)
|
|
71
|
+
}
|
|
72
|
+
puts "== #{@name} #{summary.inspect}"
|
|
73
|
+
|
|
25
74
|
results
|
|
26
75
|
end
|
|
76
|
+
|
|
77
|
+
private
|
|
78
|
+
|
|
79
|
+
# Loads prompts from:
|
|
80
|
+
# - Array<String>
|
|
81
|
+
# - .jsonl (expects objects with "prompt")
|
|
82
|
+
# - .yml/.yaml (expects key "prompts": either array of strings or a file path)
|
|
83
|
+
# - other text file (one prompt per line)
|
|
84
|
+
def load_prompts(source)
|
|
85
|
+
return source if source.is_a?(Array)
|
|
86
|
+
|
|
87
|
+
case File.extname(source)
|
|
88
|
+
when ".jsonl"
|
|
89
|
+
File.readlines(source, chomp: true).map { |l| JSON.parse(l).fetch("prompt") }
|
|
90
|
+
when ".yml", ".yaml"
|
|
91
|
+
y = YAML.load_file(source)
|
|
92
|
+
prompts = y["prompts"] || y[:prompts]
|
|
93
|
+
if prompts.is_a?(Array)
|
|
94
|
+
prompts.map(&:to_s)
|
|
95
|
+
elsif prompts.is_a?(String)
|
|
96
|
+
resolved = File.expand_path(prompts, File.dirname(source))
|
|
97
|
+
load_prompts(resolved)
|
|
98
|
+
else
|
|
99
|
+
raise ArgumentError, "YAML must contain 'prompts' (array or path), got: #{prompts.inspect}"
|
|
100
|
+
end
|
|
101
|
+
else
|
|
102
|
+
# treat as plain text: one prompt per line
|
|
103
|
+
File.readlines(source, chomp: true)
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def pctile(arr, p)
|
|
108
|
+
return nil if arr.empty?
|
|
109
|
+
a = arr.sort
|
|
110
|
+
k = (p / 100.0) * (a.length - 1)
|
|
111
|
+
f = k.floor
|
|
112
|
+
c = k.ceil
|
|
113
|
+
return a[f] if f == c
|
|
114
|
+
a[f] + (a[c] - a[f]) * (k - f)
|
|
115
|
+
end
|
|
27
116
|
end
|
|
28
117
|
end
|
|
29
118
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: raimei-bench
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Hal Fulton
|
|
@@ -9,18 +9,23 @@ bindir: exe
|
|
|
9
9
|
cert_chain: []
|
|
10
10
|
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
11
|
dependencies: []
|
|
12
|
-
description:
|
|
12
|
+
description: Run prompts against multiple backends; record p50/p95 latency & output
|
|
13
|
+
JSON reports. Experimental; APIs may change.
|
|
13
14
|
email:
|
|
14
15
|
- rubyhacker@gmail.com
|
|
15
|
-
executables:
|
|
16
|
+
executables:
|
|
17
|
+
- raimei-bench
|
|
16
18
|
extensions: []
|
|
17
19
|
extra_rdoc_files: []
|
|
18
20
|
files:
|
|
19
21
|
- CHANGELOG.md
|
|
22
|
+
- Gemfile
|
|
20
23
|
- LICENSE.txt
|
|
21
24
|
- README.md
|
|
22
25
|
- Rakefile
|
|
26
|
+
- exe/raimei-bench
|
|
23
27
|
- lib/raimei/bench.rb
|
|
28
|
+
- lib/raimei/bench/report.rb
|
|
24
29
|
- lib/raimei/bench/version.rb
|
|
25
30
|
- sig/raimei/bench.rbs
|
|
26
31
|
homepage: https://github.com/raimei-ruby/raimei
|
|
@@ -36,7 +41,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
36
41
|
requirements:
|
|
37
42
|
- - ">="
|
|
38
43
|
- !ruby/object:Gem::Version
|
|
39
|
-
version: 3.2
|
|
44
|
+
version: '3.2'
|
|
40
45
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
41
46
|
requirements:
|
|
42
47
|
- - ">="
|
|
@@ -45,5 +50,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
45
50
|
requirements: []
|
|
46
51
|
rubygems_version: 3.7.1
|
|
47
52
|
specification_version: 4
|
|
48
|
-
summary: Provider-agnostic benchmark harness
|
|
53
|
+
summary: Provider-agnostic AI benchmark harness.
|
|
49
54
|
test_files: []
|