kaba 0.3.2 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cf8ec2d335097ccfe7a540e2d2fa7253d181db9a255517fd32118707c3bbee0d
-  data.tar.gz: f6d150a528511f20b0523a1c1e30268c526386ed3d309b3faa6ff524cb2b7659
+  metadata.gz: 0add8ecf9ac4613decb084b678c044809896db95136b59e24602ae47a426d448
+  data.tar.gz: 2dda3c5fa0152b4928c4a2fadc11b18ed429cc724e87c5b38c794083cb731c69
 SHA512:
-  metadata.gz: c2fa2925775abd5bc7e04e0ff58ed3066f321dbe6c796f40eb66048945f8b3860ab8ee721b2aa4e612fac7b79f3dc57880c7b35e2e4e0ba29cc01de72db894b7
-  data.tar.gz: ba270bbcb84f5e89ad5154e1a789515c5200cf92dacaab1252f2bc60440e93d1c1fe999eaad98d9c2eb29bf73b720bb27608f22fff62f1d3d4eec52c9b5aefc1
+  metadata.gz: 622be2e55740919481236e525f2c2045f9f633eccfa36363a55d959c8d25cd859cfc349e6668df429571052d9501077f8068c35e1a72bac25ced650188c3407e
+  data.tar.gz: 56c1a740604cb48c6bfb14f5c93854597fa53922c2befd03e58cf192d29e2b2b1ac87d9480255ae6c6ef175d2652299b67b0cba8cb1e0deccf405adce9efa22b

data/README.md

@@ -30,4 +30,15 @@ alias kaba='docker run -it --rm -v "${PWD}:/workdir" ghcr.io/mjason/kaba:latest'
 ## 关联项目
 - [lisa_typechat_server](https://github.com/mjason/lisa_typechat_server)
 
-如果要修改服务地址你有两个方式，一个通过 `.env` 来处理，还有就是自己设置环境变量，变量名 `LISA_TYPECHAT_ENDPOINT`
+如果要修改服务地址你有两个方式，一个通过 `.env` 来处理，还有就是自己设置环境变量，变量名 `LISA_TYPECHAT_ENDPOINT`
+
+## changelog
+
+.env 需要更新
+```
+; LISA_TYPECHAT_ENDPOINT=https://lisa-typechat.listenai.com
+LISA_ACCESS_TOKEN=聆思平台的KEY
+
+JUDGE_ACCCESS_TOKEN=可以和LISA_ACCESS_TOKEN
+JUDGE_LLM_URI_BASE=如果需要其他提供商可以填，默认不填
+```

data/lib/kaba/application.rb

@@ -19,6 +19,17 @@ class Application
       end
     end
 
+    def judge_llm_client
+      @judge_llm_client ||= OpenAI::Client.new(
+        log_errors: true,
+        access_token: env!("JUDGE_ACCCESS_TOKEN"),
+        request_timeout: ENV.fetch("LISA_LLM_REQUEST_TIMEOUT", 120).to_i,
+        uri_base: ENV.fetch("JUDGE_LLM_URI_BASE", "https://api.listenai.com")
+      ) do |faraday|
+        faraday.adapter Faraday.default_adapter, clients: Async::HTTP::Faraday::PersistentClients
+      end
+    end
+
     def llm_client_extra_headers=(headers)
       OpenAI.configure do |config|
         config.extra_headers = headers

data/lib/kaba/test_runner.rb

@@ -26,7 +26,8 @@ class TestRunner
     model: 'spark-general-4.0', 
     judge_model: 'spark-general-4.0',  
     judge_temperature: 0.1, 
-    temperature: 0.1
+    temperature: 0.1,
+    semaphore_limit: 5
     )
 
     progressbar = TTY::ProgressBar.new(
@@ -37,8 +38,9 @@ class TestRunner
     progressbar.start
 
     Async do
+      semaphore = Async::Semaphore.new(semaphore_limit)
       _each(limit: limit) do |row|
-        Async do |task|
+        semaphore.async do |task|
           input = @prompt.render(File.read row.input_file)
 
           target = <<~Markdown
@@ -64,7 +66,7 @@ class TestRunner
           @type_right_total += 1 if type_check_response["success"]
           
           judge_input = Judge.new(input: input, output: output, target: target).render
-          judge_response = Application.llm_client.chat(
+          judge_response = Application.judge_llm_client.chat(
             parameters: {
               model: judge_model,
               messages: [ { role: 'user', content: judge_input } ],

data/lib/kaba/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kaba
-  VERSION = "0.3.2"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kaba
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
 platform: ruby
 authors:
 - MJ
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-11-16 00:00:00.000000000 Z
+date: 2024-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: async