kaba 0.2.3 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0819a2ca5b94fd6ce32a48fc973dd86d0c2234a073bf32ee9327e02f572b45b2'
-  data.tar.gz: 203c7c8cfcc4d1d059c01b01467f8b897580c4422d25db85335a2b119e674401
+  metadata.gz: cf8ec2d335097ccfe7a540e2d2fa7253d181db9a255517fd32118707c3bbee0d
+  data.tar.gz: f6d150a528511f20b0523a1c1e30268c526386ed3d309b3faa6ff524cb2b7659
 SHA512:
-  metadata.gz: 8abc31a51664418d2c8df17d2ff370c71b3124df3eb4cd17b9f05d3162e75b7fe76945986e47ae6415d0d43147be83c9627a41e9b9f7181623dcb752e91a0873
-  data.tar.gz: cc4078a6d7b4205d4403a4451ec95d4020c354809332788c755f0d2a2f6381f3d36d5206f96f7b04312a7ddf72348c2fadcf8ee519e63be4bcd730a76761b3bd
+  metadata.gz: c2fa2925775abd5bc7e04e0ff58ed3066f321dbe6c796f40eb66048945f8b3860ab8ee721b2aa4e612fac7b79f3dc57880c7b35e2e4e0ba29cc01de72db894b7
+  data.tar.gz: ba270bbcb84f5e89ad5154e1a789515c5200cf92dacaab1252f2bc60440e93d1c1fe999eaad98d9c2eb29bf73b720bb27608f22fff62f1d3d4eec52c9b5aefc1

data/exe/kaba

@@ -2,22 +2,18 @@
 require "bundler/setup"
 
 require 'json'
-require "kaba"
-
 require 'dotenv'
 Dotenv.load
 
-class Application
-  class << self
-    def connection
-      endpoint = ENV["LISA_TYPECHAT_ENDPOINT"] || "https://lisa-typechat.listenai.com"
-      @connection ||= Faraday.new(endpoint) do |faraday|
-        faraday.adapter :async_http, clients: Async::HTTP::Faraday::PersistentClients
-        faraday.request :json
-      end
-    end
-  end
-end
+require "kaba"
 
-# 运行 DPodfile 文件，DPodfile 是一个 Ruby 文件
-load DatasetSource.podfile
+if ARGV[0] == 'test'
+  # 运行测试程序
+  load DatasetSource.testfile
+elsif ARGV[0] == 'version'
+  # 显示版本号
+  puts Kaba::VERSION
+else
+  # 运行 DPodfile 文件，DPodfile 是一个 Ruby 文件
+  load DatasetSource.podfile
+end

data/kaba.gemspec

@@ -12,6 +12,7 @@ Gem::Specification.new do |spec|
   spec.description = "用来做数据集的工具"
   spec.homepage = "https://github.com/mjason/kaba.git"
   spec.required_ruby_version = ">= 3.3.0"
+  spec.license = "MIT"
 
   # spec.metadata["allowed_push_host"] = "TODO: Set to your gem server 'https://example.com'"
 
@@ -37,6 +38,8 @@ Gem::Specification.new do |spec|
   spec.add_dependency "colorize", "~> 1.1"
   spec.add_dependency "tty-progressbar", "~> 0.18.3"
   spec.add_dependency "dotenv", "~> 3.1"
+  spec.add_dependency "ruby-openai", "~> 7.3"
+  spec.add_dependency "json-repair", "~> 0.2.0"
 
   # For more information and examples about making a new gem, check out our
   # guide at: https://bundler.io/guides/creating_gem.html

data/lib/kaba/_DPodfile_

@@ -1,8 +1,6 @@
 ## 使用 Ruby 语言编写的数据集校验脚本
-# binding.irb 断点调试
 # 使用 colorize 来输出带颜色的信息，https://github.com/fazibear/colorize
 # 使用 progressbar 来显示进度条，https://github.com/piotrmurach/tty-progressbar
-# 设置数据集目录, 如果使用 Docker 方式运行，需要将数据集挂载到 /data 目录下，DatasetSource 会自动加载 /data 目录下的数据集
 source = DatasetSource.new(File.join(__dir__, 'data'))
 schema = source.schema.join('resume.ts').read
 type_name = 'Resume'

data/lib/kaba/_DTestfile_

@@ -0,0 +1,24 @@
+## 使用 Ruby 语言编写的数据集校验脚本
+# 使用 colorize 来输出带颜色的信息，https://github.com/fazibear/colorize
+# 使用 progressbar 来显示进度条，https://github.com/piotrmurach/tty-progressbar
+
+source = DatasetSource.new(File.join(__dir__, 'data'))
+schema = source.schema.join('resume.ts').read
+type_name = 'Resume'
+prompt = Prompt.new(schema, type_name)
+validate = Validate.new(schema: schema, type_name: type_name)
+
+
+test_runner = TestRunner.new(
+  source.test, 
+  schema: schema, 
+  type_name: type_name, 
+  prompt: prompt,
+  validate: validate
+)
+
+test_runner.scan(
+  limit: 1,
+)
+
+test_runner.save(source.join('report.html'))

data/lib/kaba/application.rb

@@ -0,0 +1,32 @@
+class Application
+  class << self
+    def connection
+      endpoint = ENV["LISA_TYPECHAT_ENDPOINT"] || "https://lisa-typechat.listenai.com"
+      @connection ||= Faraday.new(endpoint) do |faraday|
+        faraday.adapter :async_http, clients: Async::HTTP::Faraday::PersistentClients
+        faraday.request :json
+      end
+    end
+
+    def llm_client
+      @llm_client ||= OpenAI::Client.new(
+        log_errors: true,
+        access_token: env!("LISA_ACCESS_TOKEN"),
+        request_timeout: ENV.fetch("LISA_LLM_REQUEST_TIMEOUT", 120).to_i,
+        uri_base: ENV.fetch("LISA_LLM_URI_BASE", "https://api.listenai.com")
+      ) do |faraday|
+        faraday.adapter Faraday.default_adapter, clients: Async::HTTP::Faraday::PersistentClients
+      end
+    end
+
+    def llm_client_extra_headers=(headers)
+      OpenAI.configure do |config|
+        config.extra_headers = headers
+      end
+    end
+
+    def env!(name)
+      ENV[name] or raise "missing environment variable: #{name}"
+    end
+  end
+end

data/lib/kaba/dataset.rb

@@ -36,7 +36,9 @@ class Dataset
   end
 
   def scan(limit: nil)
-    progressbar = TTY::ProgressBar.new("Dataset: [:bar] :percent :current/:total", total: @data_files.size)
+    progressbar = TTY::ProgressBar.new(
+      "Dataset: [:bar] :percent :current/:total", 
+      total: @data_files.first(limit || @data_files.size).size)
     Async do
       _each(limit: limit) do |row, ds|
         Async do
@@ -57,12 +59,4 @@ class Dataset
     end.wait
   end
 
-end
-
-class Row
-  attr_reader :target_path, :input_file
-  def initialize(file)
-    @target_path = File.expand_path(file)
-    @input_file = @target_path.sub(/\.target\.json$/, '.input.txt')
-  end
 end

data/lib/kaba/dataset_source.rb

@@ -5,7 +5,7 @@ class DatasetSource
     @path = path
   end
 
-  [:row, :schema].each do |method_name|
+  [:row, :schema, :test].each do |method_name|
     define_method(method_name) do
       self.class.new(File.join(@path, method_name.to_s))
     end
@@ -29,13 +29,22 @@ class DatasetSource
 
   class << self
     def podfile
-      d_podfile_path = File.join(Dir.pwd, 'DPodfile')
+      d_podfile_path = File.join(Dir.pwd, 'DPodfile.rb')
       unless File.exist?(d_podfile_path)
         FileUtils.cp(File.join(__dir__, '_DPodfile_'), d_podfile_path)
       end
       d_podfile_path
     end
+
+    def testfile
+      d_testfile_path = File.join(Dir.pwd, 'DTestfile.rb')
+      unless File.exist?(d_testfile_path)
+        FileUtils.cp(File.join(__dir__, '_DTestfile_'), d_testfile_path)
+      end
+      d_testfile_path
+    end
   end
 
+
 end
 

data/lib/kaba/json.rb

@@ -0,0 +1,15 @@
+module JSON
+
+  def self.parse_llm_response(response_text)
+    start_index = response_text.index('{')
+    end_index = response_text.rindex('}')
+
+    unless start_index && end_index && end_index > start_index
+      raise "Invalid JSON response: #{response_text}"
+    end
+
+    json_text = response_text[start_index..end_index]
+    JSON.parse JSON.repair(json_text)
+  end
+  
+end

data/lib/kaba/judge.md.erb

@@ -0,0 +1,46 @@
+【系统】
+请作为一个公正的裁判，评估下面给定用户问题的AI助手所提供回答的质量。您的评估应该考虑以下因素：
+* 理解：仅考虑回答的扣题程度，不考虑回答的正确性。
+  * 核心需求是否理解；
+  * 非核心需求是否理解；
+* 生成：考虑（1）回答和问题的相关性、（2）生成文本的质量。
+  * 核心需求是否体现在答案里；
+  * 核心需求体现在答案，但是否正确实现。
+* 逻辑：考虑回答的逻辑正确性与一致性
+  * 创作/问答的逻辑主要指的是行文逻辑、发展逻辑、论证逻辑等；
+* 信息处理/代码/数学计算/逻辑推理的逻辑包括推理/计算步骤与答案正确性;
+  * 事实：前提是符合中国的国情和政治立场、法律法规和文化价值观要准确，主要指回答问题涉及的外部客观事实正确性，回复提供的信息要准确、真实、可靠、有帮助。
+  * 指令遵循：回答是否严格遵循用户问题的要求，比如是否提供了所有要求的信息，要按照给定样例格式输出回答，遇到选择或分类题应当直接输出答案而不用补充说明。
+请帮助我评估AI助手回答的好坏并给出对应的0到10得分，最终只需要给出一个综合的得分。
+【用户的问题】
+ 
+{
+    "input": "<%= @input %>",
+}
+
+【参考的回答】
+
+[
+    {
+        "target": "<%= @target %>"
+    }
+]
+
+【助手的回答】
+
+[
+    {
+        "output": "<%= @output %>"
+    }
+]
+
+【输出格式】
+
+{
+    "reason": "",
+    "score": ""
+}
+
+请注意区分您的最终任务和用户问题中提出的任务，最终的任务是完成评估打分任务，而不要直接回答给定的用户问题。
+请按照输出格式给出评分理由和助手回答的得分，不要输出json格式外的内容。
+【评估结果】

data/lib/kaba/judge.rb

@@ -0,0 +1,23 @@
+require 'erb'
+
+class Judge
+  def initialize(input: , target: , output:)
+    @input = input
+    @target = target
+    @output = output
+  end
+
+  def render
+    ERB.new(File.read(self.class.prompt_path)).result(binding)
+  end
+
+  class << self
+    def prompt_path
+      @prompt_path || File.join(__dir__, 'judge.md.erb')
+    end
+
+    def set_prompt_path(path)
+      @prompt_path = path
+    end
+  end
+end

data/lib/kaba/prompt.rb

@@ -13,9 +13,9 @@ class Prompt
     request_body = {
       schema: schema,
       typeName: @type_name,
-      inpu: input
+      input: input
     }
-    Application.connection.post('/prompt', request_body).body
+    resp = Application.connection.post('/prompt', request_body).body
   end
 
   class << self

data/lib/kaba/report.html.erb

@@ -0,0 +1,125 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>Report</title>
+  <style>
+    body {
+      font-family: Arial, sans-serif;
+      margin: 20px;
+      line-height: 1.6;
+    }
+    .header {
+      margin-bottom: 20px;
+    }
+    .header h1 {
+      font-size: 24px;
+    }
+    .metrics {
+      font-size: 18px;
+      margin-bottom: 10px;
+    }
+    .line-item {
+      margin-bottom: 20px;
+      border: 1px solid #ddd;
+      border-radius: 5px;
+      background-color: #f9f9f9;
+    }
+    .line-item h2 {
+      font-size: 18px;
+      margin: 0;
+      padding: 10px;
+      background-color: #f0f0f0;
+      cursor: pointer;
+    }
+    .line-item h2 .status {
+      font-size: 14px;
+      color: #666;
+      margin-left: 10px;
+    }
+    .line-item .content {
+      display: none;
+      padding: 10px;
+    }
+    .line-item pre {
+      background-color: #282c34;
+      color: #abb2bf;
+      padding: 10px;
+      overflow-x: auto;
+      border-radius: 5px;
+    }
+  </style>
+  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.7.0/styles/default.min.css">
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.7.0/highlight.min.js"></script>
+  <script>
+    document.addEventListener("DOMContentLoaded", () => {
+      // Initialize Highlight.js
+      hljs.highlightAll();
+
+      // Add toggle functionality
+      document.querySelectorAll(".line-item h2").forEach(header => {
+        header.addEventListener("click", () => {
+          const content = header.nextElementSibling;
+          content.style.display = content.style.display === "none" ? "block" : "none";
+        });
+      });
+    });
+  </script>
+</head>
+<body>
+  <div class="header">
+    <h1>Test Report</h1>
+    <div class="metrics">
+      <p><strong>测试总数: </strong><%= @lines.size %></p>
+      <p><strong>类型测试通过: </strong><%= @type_right_total %></p>
+      <p><strong>平均分: </strong><%= (@score_total.to_f / @lines.size).round(2) %></p>
+    </div>
+  </div>
+
+  <div class="content">
+    <% @lines.each_with_index do |line, index| %>
+      <div class="line-item">
+        <h2>
+          测试路径: <%= line[:row].input_file %>
+          <span class="status">
+            <%= line[:type_check_response]["success"] ? "✅ 定义检查" : "❌ 定义检查" %> | 得分: <%= line[:judge_json]["score"] %>
+          </span>
+        </h2>
+
+        <div class="content">
+          <div class="section">
+            <strong>AI裁判输出:</strong>
+            <pre><code class="language-json"><%= JSON.pretty_generate line[:judge_json] %></code></pre>
+          </div>
+
+          <div class="section">
+            <strong>类型检查:</strong>
+            <pre><code class="language-json"><%= JSON.pretty_generate(line[:type_check_response]) %></code></pre>
+          </div>
+
+          <div class="section">
+            <strong>Prompt:</strong>
+            <pre><code class="language-markdown"><%= line[:input] %></code></pre>
+          </div>
+
+          <div class="section">
+            <strong>原始输出:</strong>
+            <pre><code class="language-markdown"><%= line[:output] %></code></pre>
+          </div>
+
+          <div class="section">
+            <strong>格式化输出:</strong>
+            <pre><code class="language-json"><%= JSON.pretty_generate(line[:output_json]) %></code></pre>
+          </div>
+
+          <div class="section">
+            <strong>目标结果:</strong>
+            <pre><code class="language-json"><%= line[:target] %></code></pre>
+          </div>
+        </div>
+      </div>
+    <% end %>
+  </div>
+</body>
+</html>

data/lib/kaba/row.rb

@@ -0,0 +1,7 @@
+class Row
+  attr_reader :target_path, :input_file
+  def initialize(file)
+    @target_path = File.expand_path(file)
+    @input_file = @target_path.sub(/\.target\.json$/, '.input.txt')
+  end
+end

data/lib/kaba/test_runner.rb

@@ -0,0 +1,112 @@
+require 'erb'
+
+class TestRunner
+  def initialize(path, schema:, type_name:, prompt:, validate:)
+
+    @test_files = Dir.glob(File.join(File.expand_path(path), '*.target.json'))
+    @lines = []
+    @schema = schema
+    @type_name = type_name
+    @prompt = prompt || Prompt.new(@schema, @type_name)
+    @validate = validate || Validate.new(schema: @schema, type_name: @type_name)
+
+    @type_right_total = 0
+    @score_total = 0
+
+  end
+
+  def _each(limit: nil)
+    @test_files.first(limit || @test_files.size).each do |file|
+      yield(Row.new(file), self)
+    end
+  end
+
+  def scan(
+    limit: nil, 
+    model: 'spark-general-4.0', 
+    judge_model: 'spark-general-4.0',  
+    judge_temperature: 0.1, 
+    temperature: 0.1
+    )
+
+    progressbar = TTY::ProgressBar.new(
+      "Test: [:bar] :percent :current/:total",
+       total: @test_files.first(limit || @test_files.size).size
+    )
+
+    progressbar.start
+
+    Async do
+      _each(limit: limit) do |row|
+        Async do |task|
+          input = @prompt.render(File.read row.input_file)
+
+          target = <<~Markdown
+          ```json
+          #{JSON.pretty_generate(JSON.parse(File.read(row.target_path)))}
+          ```
+          Markdown
+          output = ""
+          Application.llm_client.chat(
+            parameters: {
+              model: model,
+              messages: [ { role: 'user', content: input } ],
+              temperature: temperature,
+              stream: proc do |chunk, _bytesize|
+                output += chunk.dig("choices", 0, "delta", "content")
+              end
+            }
+          )
+
+          output_json = JSON.parse_llm_response output
+
+          type_check_response = JSON.parse @validate.run(output_json).body
+          @type_right_total += 1 if type_check_response["success"]
+          
+          judge_input = Judge.new(input: input, output: output, target: target).render
+          judge_response = Application.llm_client.chat(
+            parameters: {
+              model: judge_model,
+              messages: [ { role: 'user', content: judge_input } ],
+              temperature: judge_temperature,
+            }
+          ).dig("choices", 0, "message", "content")
+
+          judge_json = JSON.parse_llm_response judge_response
+          @score_total += judge_json["score"].to_i
+
+          @lines << {
+            row: row,
+            input: input,
+            output: output,
+            output_json: output_json,
+            type_check_response: type_check_response,
+            target: target,
+            judge_response: judge_response,
+            judge_json: judge_json,
+          }
+
+          progressbar.advance
+        end
+      end
+    end.wait
+  end
+
+  def save(file_path)
+    File.open(File.expand_path(file_path), 'w') do |file|
+      file.puts ERB.new(File.read(self.class.report_template_path)).result(binding)
+    end
+  end
+
+
+  class << self
+    def report_template_path
+      @report_template_path || File.join(__dir__, 'report.html.erb')
+    end
+
+    def report_template_path=(path)
+      @report_template_path = path
+    end
+  end
+
+end

data/lib/kaba/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kaba
-  VERSION = "0.2.3"
+  VERSION = "0.3.2"
 end

data/lib/kaba.rb

@@ -5,12 +5,20 @@ require 'faraday'
 require 'colorize'
 require 'tty-progressbar'
 require 'async/http/faraday'
+require 'openai'
+require 'json/repair'
 
+require_relative "kaba/application"
+require_relative "kaba/json"
+
+require_relative "kaba/row"
 require_relative "kaba/version"
 require_relative "kaba/dataset"
 require_relative "kaba/dataset_source"
 require_relative "kaba/prompt"
 require_relative "kaba/validate"
+require_relative "kaba/judge"
+require_relative "kaba/test_runner"
 
 module Kaba
   class Error < StandardError; end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kaba
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.2
 platform: ruby
 authors:
 - MJ
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-11-14 00:00:00.000000000 Z
+date: 2024-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: async
@@ -94,6 +94,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: ruby-openai
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.3'
+- !ruby/object:Gem::Dependency
+  name: json-repair
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
 description: 用来做数据集的工具
 email:
 - tywf91@gmail.com
@@ -110,14 +138,23 @@ files:
 - kaba.gemspec
 - lib/kaba.rb
 - lib/kaba/_DPodfile_
+- lib/kaba/_DTestfile_
+- lib/kaba/application.rb
 - lib/kaba/dataset.rb
 - lib/kaba/dataset_source.rb
+- lib/kaba/json.rb
+- lib/kaba/judge.md.erb
+- lib/kaba/judge.rb
 - lib/kaba/prompt.rb
+- lib/kaba/report.html.erb
+- lib/kaba/row.rb
+- lib/kaba/test_runner.rb
 - lib/kaba/validate.rb
 - lib/kaba/version.rb
 - sig/kaba.rbs
 homepage: https://github.com/mjason/kaba.git
-licenses: []
+licenses:
+- MIT
 metadata:
   homepage_uri: https://github.com/mjason/kaba.git
   source_code_uri: https://github.com/mjason/kaba.git