kaba 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 87911927cc961d9cde4f12b9ec86da34b1e13b00fa0c984a69c758ff06bdf67f
-  data.tar.gz: 8da74f6aeb020d0438e33f69387c975643ec72739d029a193370b7d3d9878e03
+  metadata.gz: 17f06505d496fef06c186773df25fa37d7a2e465fac06682f51a300933e286d5
+  data.tar.gz: caefca3868f25c1cd15e6bf441ae3e595a2be4b5fb7574431c1a57284ca00327
 SHA512:
-  metadata.gz: 5271d2d31f23d185ecfe7b3d0fc47b7f200a1d659d6e1ca8939a9216e5ead2449dc1abdddb622c3d972fb3b5b668cc481e0626425454bbfd03807e8432f51aec
-  data.tar.gz: 4ba6e7b05b0fffc61f6462188809d63bb7a700876b4084e82ecce1096c8b5c5ab15fef7b82c33884513cc2f999986855f3f779c1bb3d3acc9b1c90ef79536e62
+  metadata.gz: '09d533d49ac00fd23230ae8ea81f2c189d81b285738142e323b07bb5ea9cbac3cd91ae8f41daa56c37ea1c5297d99573e8dac2c411ec5add299644e8d888ea1e'
+  data.tar.gz: 15751ae8ef5e041b5ac8638626bac0e938fe989a5647af4c59c8a9cdc803e0796889f4353adf0a5fefe0af0430187f82ca21ccdd9dcd0a53c88a3edd8829c45d

data/exe/kaba

@@ -1,29 +1,19 @@
 #!/usr/bin/env ruby
 require "bundler/setup"
 
-require 'async'
-require 'faraday'
-require 'colorize'
-require 'tty-progressbar'
-require 'async/http/faraday'
-
 require 'json'
-require "kaba"
-
 require 'dotenv'
 Dotenv.load
 
-class Application
-  class << self
-    def connection
-      endpoint = ENV["LISA_TYPECHAT_ENDPOINT"] || "https://lisa-typechat.listenai.com"
-      @connection ||= Faraday.new(endpoint) do |faraday|
-        faraday.adapter :async_http, clients: Async::HTTP::Faraday::PersistentClients
-        faraday.request :json
-      end
-    end
-  end
-end
+require "kaba"
 
-# 运行 DPodfile 文件，DPodfile 是一个 Ruby 文件
-load DatasetSource.podfile
+if ARGV[0] == 'test'
+  # 运行测试程序
+  load DatasetSource.testfile
+elsif ARGV[0] == 'version'
+  # 显示版本号
+  puts Kaba::VERSION
+else
+  # 运行 DPodfile 文件，DPodfile 是一个 Ruby 文件
+  load DatasetSource.podfile
+end

data/kaba.gemspec

@@ -37,6 +37,8 @@ Gem::Specification.new do |spec|
   spec.add_dependency "colorize", "~> 1.1"
   spec.add_dependency "tty-progressbar", "~> 0.18.3"
   spec.add_dependency "dotenv", "~> 3.1"
+  spec.add_dependency "ruby-openai", "~> 7.3"
+  spec.add_dependency "json-repair", "~> 0.2.0"
 
   # For more information and examples about making a new gem, check out our
   # guide at: https://bundler.io/guides/creating_gem.html

data/lib/kaba/_DPodfile_

@@ -1,8 +1,6 @@
 ## 使用 Ruby 语言编写的数据集校验脚本
-# binding.irb 断点调试
 # 使用 colorize 来输出带颜色的信息，https://github.com/fazibear/colorize
 # 使用 progressbar 来显示进度条，https://github.com/piotrmurach/tty-progressbar
-# 设置数据集目录, 如果使用 Docker 方式运行，需要将数据集挂载到 /data 目录下，DatasetSource 会自动加载 /data 目录下的数据集
 source = DatasetSource.new(File.join(__dir__, 'data'))
 schema = source.schema.join('resume.ts').read
 type_name = 'Resume'
@@ -24,6 +22,8 @@ validate.run_files(source.row)
 dataset.scan()
 dataset.save(source.join('train.jsonl'))
 
+puts "Dataset 校验结果：#{dataset.validate}"
+
 ## 高级玩法，不要轻易尝试
 #
 ### 可以加入 limit 来限制读取的文件数量，validate.run_files('./data/row', limit: 1) do |response, json, file| 

data/lib/kaba/_DTestfile_

@@ -0,0 +1,24 @@
+## 使用 Ruby 语言编写的数据集校验脚本
+# 使用 colorize 来输出带颜色的信息，https://github.com/fazibear/colorize
+# 使用 progressbar 来显示进度条，https://github.com/piotrmurach/tty-progressbar
+
+source = DatasetSource.new(File.join(__dir__, 'data'))
+schema = source.schema.join('resume.ts').read
+type_name = 'Resume'
+prompt = Prompt.new(schema, type_name)
+validate = Validate.new(schema: schema, type_name: type_name)
+
+
+test_runner = TestRunner.new(
+  source.test, 
+  schema: schema, 
+  type_name: type_name, 
+  prompt: prompt,
+  validate: validate
+)
+
+test_runner.scan(
+  limit: 1,
+)
+
+test_runner.save(source.join('report.html'))

data/lib/kaba/application.rb

@@ -0,0 +1,31 @@
+class Application
+  class << self
+    def connection
+      endpoint = ENV["LISA_TYPECHAT_ENDPOINT"] || "https://lisa-typechat.listenai.com"
+      @connection ||= Faraday.new(endpoint) do |faraday|
+        faraday.adapter :async_http, clients: Async::HTTP::Faraday::PersistentClients
+        faraday.request :json
+      end
+    end
+
+    def llm_client
+      @llm_client ||= OpenAI::Client.new(
+        access_token: env!("LISA_ACCESS_TOKEN"),
+        request_timeout: ENV.fetch("LISA_LLM_REQUEST_TIMEOUT", 120).to_i,
+        uri_base: ENV.fetch("LISA_LLM_URI_BASE", "https://api.listenai.com")
+      ) do |faraday|
+        faraday.adapter :async_http, clients: Async::HTTP::Faraday::PersistentClients
+      end
+    end
+
+    def llm_client_extra_headers=(headers)
+      OpenAI.configure do |config|
+        config.extra_headers = headers
+      end
+    end
+
+    def env!(name)
+      ENV[name] or raise "missing environment variable: #{name}"
+    end
+  end
+end

data/lib/kaba/dataset.rb

@@ -36,7 +36,9 @@ class Dataset
   end
 
   def scan(limit: nil)
-    progressbar = TTY::ProgressBar.new("Dataset: [:bar] :percent :current/:total", total: @data_files.size)
+    progressbar = TTY::ProgressBar.new(
+      "Dataset: [:bar] :percent :current/:total", 
+      total: @data_files.first(limit || @data_files.size).size)
     Async do
       _each(limit: limit) do |row, ds|
         Async do
@@ -57,12 +59,4 @@ class Dataset
     end.wait
   end
 
-end
-
-class Row
-  attr_reader :target_path, :input_file
-  def initialize(file)
-    @target_path = File.expand_path(file)
-    @input_file = @target_path.sub(/\.target\.json$/, '.input.txt')
-  end
 end

data/lib/kaba/dataset_source.rb

@@ -5,7 +5,7 @@ class DatasetSource
     @path = path
   end
 
-  [:row, :schema].each do |method_name|
+  [:row, :schema, :test].each do |method_name|
     define_method(method_name) do
       self.class.new(File.join(@path, method_name.to_s))
     end
@@ -29,13 +29,22 @@ class DatasetSource
 
   class << self
     def podfile
-      d_podfile_path = File.join(Dir.pwd, 'DPodfile')
+      d_podfile_path = File.join(Dir.pwd, 'DPodfile.rb')
       unless File.exist?(d_podfile_path)
         FileUtils.cp(File.join(__dir__, '_DPodfile_'), d_podfile_path)
       end
       d_podfile_path
     end
+
+    def testfile
+      d_testfile_path = File.join(Dir.pwd, 'DTestfile.rb')
+      unless File.exist?(d_testfile_path)
+        FileUtils.cp(File.join(__dir__, '_DTestfile_'), d_testfile_path)
+      end
+      d_testfile_path
+    end
   end
 
+
 end
 

data/lib/kaba/json.rb

@@ -0,0 +1,15 @@
+module JSON
+
+  def self.parse_llm_response(response_text)
+    start_index = response_text.index('{')
+    end_index = response_text.rindex('}')
+
+    unless start_index && end_index && end_index > start_index
+      raise "Invalid JSON response: #{response_text}"
+    end
+
+    json_text = response_text[start_index..end_index]
+    JSON.parse JSON.repair(json_text)
+  end
+  
+end

data/lib/kaba/judge.md.erb

@@ -0,0 +1,46 @@
+【系统】
+请作为一个公正的裁判，评估下面给定用户问题的AI助手所提供回答的质量。您的评估应该考虑以下因素：
+* 理解：仅考虑回答的扣题程度，不考虑回答的正确性。
+  * 核心需求是否理解；
+  * 非核心需求是否理解；
+* 生成：考虑（1）回答和问题的相关性、（2）生成文本的质量。
+  * 核心需求是否体现在答案里；
+  * 核心需求体现在答案，但是否正确实现。
+* 逻辑：考虑回答的逻辑正确性与一致性
+  * 创作/问答的逻辑主要指的是行文逻辑、发展逻辑、论证逻辑等；
+* 信息处理/代码/数学计算/逻辑推理的逻辑包括推理/计算步骤与答案正确性;
+  * 事实：前提是符合中国的国情和政治立场、法律法规和文化价值观要准确，主要指回答问题涉及的外部客观事实正确性，回复提供的信息要准确、真实、可靠、有帮助。
+  * 指令遵循：回答是否严格遵循用户问题的要求，比如是否提供了所有要求的信息，要按照给定样例格式输出回答，遇到选择或分类题应当直接输出答案而不用补充说明。
+请帮助我评估AI助手回答的好坏并给出对应的0到10得分，最终只需要给出一个综合的得分。
+【用户的问题】
+ 
+{
+    "input": "<%= @input %>",
+}
+
+【参考的回答】
+
+[
+    {
+        "target": "<%= @target %>"
+    }
+]
+
+【助手的回答】
+
+[
+    {
+        "output": "<%= @output %>"
+    }
+]
+
+【输出格式】
+
+{
+    "reason": "",
+    "score": ""
+}
+
+请注意区分您的最终任务和用户问题中提出的任务，最终的任务是完成评估打分任务，而不要直接回答给定的用户问题。
+请按照输出格式给出评分理由和助手回答的得分，不要输出json格式外的内容。
+【评估结果】

data/lib/kaba/judge.rb

@@ -0,0 +1,23 @@
+require 'erb'
+
+class Judge
+  def initialize(input: , target: , output:)
+    @input = input
+    @target = target
+    @output = output
+  end
+
+  def render
+    ERB.new(File.read(self.class.prompt_path)).result(binding)
+  end
+
+  class << self
+    def prompt_path
+      @prompt_path || File.join(__dir__, 'judge.md.erb')
+    end
+
+    def set_prompt_path(path)
+      @prompt_path = path
+    end
+  end
+end

data/lib/kaba/prompt.rb

@@ -13,9 +13,9 @@ class Prompt
     request_body = {
       schema: schema,
       typeName: @type_name,
-      inpu: input
+      input: input
     }
-    Application.connection.post('/prompt', request_body).body
+    resp = Application.connection.post('/prompt', request_body).body
   end
 
   class << self

data/lib/kaba/report.html.erb

@@ -0,0 +1,125 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>Report</title>
+  <style>
+    body {
+      font-family: Arial, sans-serif;
+      margin: 20px;
+      line-height: 1.6;
+    }
+    .header {
+      margin-bottom: 20px;
+    }
+    .header h1 {
+      font-size: 24px;
+    }
+    .metrics {
+      font-size: 18px;
+      margin-bottom: 10px;
+    }
+    .line-item {
+      margin-bottom: 20px;
+      border: 1px solid #ddd;
+      border-radius: 5px;
+      background-color: #f9f9f9;
+    }
+    .line-item h2 {
+      font-size: 18px;
+      margin: 0;
+      padding: 10px;
+      background-color: #f0f0f0;
+      cursor: pointer;
+    }
+    .line-item h2 .status {
+      font-size: 14px;
+      color: #666;
+      margin-left: 10px;
+    }
+    .line-item .content {
+      display: none;
+      padding: 10px;
+    }
+    .line-item pre {
+      background-color: #282c34;
+      color: #abb2bf;
+      padding: 10px;
+      overflow-x: auto;
+      border-radius: 5px;
+    }
+  </style>
+  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.7.0/styles/default.min.css">
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.7.0/highlight.min.js"></script>
+  <script>
+    document.addEventListener("DOMContentLoaded", () => {
+      // Initialize Highlight.js
+      hljs.highlightAll();
+
+      // Add toggle functionality
+      document.querySelectorAll(".line-item h2").forEach(header => {
+        header.addEventListener("click", () => {
+          const content = header.nextElementSibling;
+          content.style.display = content.style.display === "none" ? "block" : "none";
+        });
+      });
+    });
+  </script>
+</head>
+<body>
+  <div class="header">
+    <h1>Test Report</h1>
+    <div class="metrics">
+      <p><strong>测试总数: </strong><%= @lines.size %></p>
+      <p><strong>类型测试通过: </strong><%= @type_right_total %></p>
+      <p><strong>平均分: </strong><%= (@score_total.to_f / @lines.size).round(2) %></p>
+    </div>
+  </div>
+
+  <div class="content">
+    <% @lines.each_with_index do |line, index| %>
+      <div class="line-item">
+        <h2>
+          测试路径: <%= line[:row].input_file %>
+          <span class="status">
+            <%= line[:type_check_response]["success"] ? "✅ 定义检查" : "❌ 定义检查" %> | 得分: <%= line[:judge_json]["score"] %>
+          </span>
+        </h2>
+
+        <div class="content">
+          <div class="section">
+            <strong>AI裁判输出:</strong>
+            <pre><code class="language-json"><%= JSON.pretty_generate line[:judge_json] %></code></pre>
+          </div>
+
+          <div class="section">
+            <strong>类型检查:</strong>
+            <pre><code class="language-json"><%= JSON.pretty_generate(line[:type_check_response]) %></code></pre>
+          </div>
+
+          <div class="section">
+            <strong>Prompt:</strong>
+            <pre><code class="language-markdown"><%= line[:input] %></code></pre>
+          </div>
+
+          <div class="section">
+            <strong>原始输出:</strong>
+            <pre><code class="language-markdown"><%= line[:output] %></code></pre>
+          </div>
+
+          <div class="section">
+            <strong>格式化输出:</strong>
+            <pre><code class="language-json"><%= JSON.pretty_generate(line[:output_json]) %></code></pre>
+          </div>
+
+          <div class="section">
+            <strong>目标结果:</strong>
+            <pre><code class="language-json"><%= line[:target] %></code></pre>
+          </div>
+        </div>
+      </div>
+    <% end %>
+  </div>
+</body>
+</html>

data/lib/kaba/row.rb

@@ -0,0 +1,7 @@
+class Row
+  attr_reader :target_path, :input_file
+  def initialize(file)
+    @target_path = File.expand_path(file)
+    @input_file = @target_path.sub(/\.target\.json$/, '.input.txt')
+  end
+end

data/lib/kaba/test_runner.rb

@@ -0,0 +1,109 @@
+require 'erb'
+
+class TestRunner
+  def initialize(path, schema:, type_name:, prompt:, validate:)
+
+    @test_files = Dir.glob(File.join(File.expand_path(path), '*.target.json'))
+    @lines = []
+    @schema = schema
+    @type_name = type_name
+    @prompt = prompt || Prompt.new(@schema, @type_name)
+    @validate = validate || Validate.new(schema: @schema, type_name: @type_name)
+
+    @type_right_total = 0
+    @score_total = 0
+
+  end
+
+  def _each(limit: nil)
+    @test_files.first(limit || @test_files.size).each do |file|
+      yield(Row.new(file), self)
+    end
+  end
+
+  def scan(
+    limit: nil, 
+    model: 'spark-general-4.0', 
+    judge_model: 'spark-general-4.0',  
+    judge_temperature: 0.1, 
+    temperature: 0.1
+    )
+
+    progressbar = TTY::ProgressBar.new(
+      "Test: [:bar] :percent :current/:total",
+       total: @test_files.first(limit || @test_files.size).size
+    )
+
+    progressbar.start
+
+    Async do
+      _each(limit: limit) do |row|
+        Async do |task|
+          input = @prompt.render(File.read row.input_file)
+
+          target = <<~Markdown
+          ```json
+          #{JSON.pretty_generate(JSON.parse(File.read(row.target_path)))}
+          ```
+          Markdown
+          output = Application.llm_client.chat(
+            parameters: {
+              model: model,
+              messages: [ { role: 'user', content: input } ],
+              temperature: temperature,
+            }
+          ).dig("choices", 0, "message", "content")
+          
+
+          output_json = JSON.parse_llm_response output
+
+          type_check_response = JSON.parse @validate.run(output_json).body
+          @type_right_total += 1 if type_check_response["success"]
+          
+          judge_input = Judge.new(input: input, output: output, target: target).render
+          judge_response = Application.llm_client.chat(
+            parameters: {
+              model: judge_model,
+              messages: [ { role: 'user', content: judge_input } ],
+              temperature: judge_temperature,
+            }
+          ).dig("choices", 0, "message", "content")
+
+          judge_json = JSON.parse_llm_response judge_response
+          @score_total += judge_json["score"].to_i
+
+          @lines << {
+            row: row,
+            input: input,
+            output: output,
+            output_json: output_json,
+            type_check_response: type_check_response,
+            target: target,
+            judge_response: judge_response,
+            judge_json: judge_json,
+          }
+
+          progressbar.advance
+        end
+      end
+    end.wait
+  end
+
+  def save(file_path)
+    File.open(File.expand_path(file_path), 'w') do |file|
+      file.puts ERB.new(File.read(self.class.report_template_path)).result(binding)
+    end
+  end
+
+
+  class << self
+    def report_template_path
+      @report_template_path || File.join(__dir__, 'report.html.erb')
+    end
+
+    def report_template_path=(path)
+      @report_template_path = path
+    end
+  end
+
+end

data/lib/kaba/validate.rb

@@ -18,7 +18,7 @@ class Validate
   # 读取某个文件然后运行
   def run_file(file)
     input = JSON.parse File.read(File.expand_path file)
-    ValidateReponse.new run(input)
+    ValidateReponse.new run(input), file: file
   end
 
   # 读取某个文件夹下的然后运行，运行有结果了 block 会被调用
@@ -75,8 +75,8 @@ class Validate
 
     def to_s
       s = "#{'success:'.colorize(:bold_blue)} #{success? ? 'true'.colorize(:green) : 'false'.colorize(:red)}"
-      s += "\n#{'file:'.colorize(:bold_blue)} #{file.colorize(:yellow)}"
-      s += "\n#{'message:'.colorize(:bold_blue)} #{message.colorize(:yellow)}" unless success?
+      s += "\n#{'file:'.colorize(:bold_blue)} #{file&.to_s&.colorize(:yellow)}"
+      s += "\n#{'message:'.colorize(:bold_blue)} #{message&.colorize(:yellow)}" unless success?
       s += "\n\n"
     end
   end

data/lib/kaba/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kaba
-  VERSION = "0.2.2"
+  VERSION = "0.3.0"
 end

data/lib/kaba.rb

@@ -1,10 +1,24 @@
 # frozen_string_literal: true
 
+require 'async'
+require 'faraday'
+require 'colorize'
+require 'tty-progressbar'
+require 'async/http/faraday'
+require 'openai'
+require 'json/repair'
+
+require_relative "kaba/application"
+require_relative "kaba/json"
+
+require_relative "kaba/row"
 require_relative "kaba/version"
 require_relative "kaba/dataset"
 require_relative "kaba/dataset_source"
 require_relative "kaba/prompt"
 require_relative "kaba/validate"
+require_relative "kaba/judge"
+require_relative "kaba/test_runner"
 
 module Kaba
   class Error < StandardError; end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kaba
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - MJ
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-11-13 00:00:00.000000000 Z
+date: 2024-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: async
@@ -94,6 +94,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: ruby-openai
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.3'
+- !ruby/object:Gem::Dependency
+  name: json-repair
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
 description: 用来做数据集的工具
 email:
 - tywf91@gmail.com
@@ -110,9 +138,17 @@ files:
 - kaba.gemspec
 - lib/kaba.rb
 - lib/kaba/_DPodfile_
+- lib/kaba/_DTestfile_
+- lib/kaba/application.rb
 - lib/kaba/dataset.rb
 - lib/kaba/dataset_source.rb
+- lib/kaba/json.rb
+- lib/kaba/judge.md.erb
+- lib/kaba/judge.rb
 - lib/kaba/prompt.rb
+- lib/kaba/report.html.erb
+- lib/kaba/row.rb
+- lib/kaba/test_runner.rb
 - lib/kaba/validate.rb
 - lib/kaba/version.rb
 - sig/kaba.rbs