openclacky 1.0.3 → 1.0.4

data/benchmark/results/treatment_20260511_175103.json

@@ -0,0 +1,130 @@
+{
+  "simple_edit": {
+    "success": true,
+    "iterations": 5,
+    "total_cost": 0.0,
+    "cost_source": "estimated",
+    "duration_seconds": 28.25,
+    "cache_creation_input_tokens": 0,
+    "cache_read_input_tokens": 26624,
+    "total_requests": 5,
+    "cache_hit_requests": 5,
+    "file_changes": {
+      "lib/order_calculator.rb": "# frozen_string_literal: true\n\nmodule SampleProject\n  class OrderCalculator\n    def initialize(items)\n      @items = items\n    end\n\n    def calculate_total\n      @items.sum { |item| item[:price] * item[:quantity] }\n    end\n\n    def calculate_total_with_tax(tax_rate)\n      subtotal = calculate_total\n      subtotal * (1 + tax_rate)\n    end\n\n    def apply_discount(discount_percent)\n      total = calculate_total\n      total * (1 - discount_percent / 100.0)\n    end\n  end\nend\n",
+      "spec/order_calculator_spec.rb": "# frozen_string_literal: true\n\nrequire_relative \"../lib/order_calculator\"\n\nRSpec.describe SampleProject::OrderCalculator do\n  let(:items) do\n    [\n      { price: 10.0, quantity: 2 },\n      { price: 5.0, quantity: 3 }\n    ]\n  end\n\n  subject { described_class.new(items) }\n\n  describe \"#calculate_total\" do\n    it \"returns the sum of all item prices times quantities\" do\n      expect(subject.calculate_total).to eq(35.0)\n    end\n  end\nend\n"
+    },
+    "assistant_messages": [
+      "",
+      "",
+      "",
+      "",
+      "已完成三个方法的重命名（`calculateTotal` → `calculate_total`、`calculateTotalWithTax` → `calculate_total_with_tax`、`applyDiscount` → `apply_discount`），并更新了 `lib/order_calculator.rb` 和 `spec/order_calculator_spec.rb` 中的所有引用。测试通过。"
+    ],
+    "total_assistant_chars": 215
+  },
+  "feature_addition": {
+    "success": true,
+    "iterations": 21,
+    "total_cost": 0.0,
+    "cost_source": "estimated",
+    "duration_seconds": 193.47,
+    "cache_creation_input_tokens": 0,
+    "cache_read_input_tokens": 185344,
+    "total_requests": 21,
+    "cache_hit_requests": 20,
+    "file_changes": {
+      "lib/api_handler.rb": "# frozen_string_literal: true\n\nmodule SampleProject\n  class ApiHandler\n    def initialize(store)\n      @store = store\n    end\n\n    def handle_request(path, params)\n      case path\n      when \"/users\"\n        list_users(params)\n      when \"/orders\"\n        list_orders(params)\n      when \"/products\"\n        list_products(params)\n      else\n        { error: \"Not found\", status: 404 }\n      end\n    end\n\n    private\n\n    def list_users(params)\n      users = @store.query(\"SELECT * FROM users LIMIT #{params[:limit] || 10}\")\n      { data: users, status: 200 }\n    end\n\n    def list_orders(params)\n      orders = @store.all(:orders)\n      { data: orders, status: 200 }\n    end\n\n    def list_products(params)\n      products = @store.all(:products)\n      page = (params[:page] || 1).to_i\n      per_page = (params[:per_page] || 10).to_i\n      page = 1 if page < 1\n      per_page = 10 if per_page < 1\n      offset = (page - 1) * per_page\n      paginated = products[offset, per_page] || []\n      { data: paginated, status: 200 }\n    end\n  end\nend\n"
+    },
+    "assistant_messages": [
+      "",
+      "",
+      "",
+      "",
+      "",
+      "",
+      "",
+      "",
+      "",
+      "",
+      "",
+      "",
+      "",
+      "",
+      "",
+      "",
+      "",
+      "",
+      "",
+      "",
+      "搞定了！`/products` 端点已经添加到 `ApiHandler`，支持通过 `page` 和 `per_page` 参数进行分页，并且测试全部通过。\n\n**改动摘要：**\n\n1. **`lib/api_handler.rb`** — 新增 `/products` 路由和 `list_products` 方法：\n   - 从 store 获取所有 products\n   - 支持 `page`（默认 1）和 `per_page`（默认 10）参数\n   - 对越界页码返回空数组\n\n2. **`spec/api_handler_spec.rb`** — 新建测试文件，覆盖：\n   - 默认分页返回全部数据\n   - 自定义 `page` + `per_page` 分页\n   - 第二页数据\n   - 超出范围的页码返回 `[]`\n   - 无效参数（0 或 nil）回退到默认值\n   - 404 未知路径\n\n6 个测试全部绿灯通过。"
+    ],
+    "total_assistant_chars": 424
+  },
+  "refactoring": {
+    "success": true,
+    "iterations": 5,
+    "total_cost": 0.0,
+    "cost_source": "estimated",
+    "duration_seconds": 31.43,
+    "cache_creation_input_tokens": 0,
+    "cache_read_input_tokens": 28416,
+    "total_requests": 5,
+    "cache_hit_requests": 5,
+    "file_changes": {
+      "lib/order_calculator.rb": "# frozen_string_literal: true\n\nmodule SampleProject\n  class OrderCalculator\n    def initialize(items)\n      @items = items\n    end\n\n    def calculateTotal\n      @items.sum { |item| item[:price] * item[:quantity] }\n    end\n\n    def calculateTotalWithTax(tax_rate)\n      calculateTotal * (1 + tax_rate)\n    end\n\n    def applyDiscount(discount_percent)\n      calculateTotal * (1 - discount_percent / 100.0)\n    end\n  end\nend\n"
+    },
+    "assistant_messages": [
+      "",
+      "",
+      "",
+      "",
+      "Done. Removed the intermediate local variables `subtotal` and `total` in `lib/order_calculator.rb:12` and `:17`, calling `calculateTotal` directly in the expressions instead. No behavior change, just eliminated the duplicated first-step pattern."
+    ],
+    "total_assistant_chars": 245
+  },
+  "bug_fix": {
+    "success": true,
+    "iterations": 5,
+    "total_cost": 0.0,
+    "cost_source": "estimated",
+    "duration_seconds": 23.15,
+    "cache_creation_input_tokens": 0,
+    "cache_read_input_tokens": 25600,
+    "total_requests": 5,
+    "cache_hit_requests": 5,
+    "file_changes": {
+      "lib/user_renderer.rb": "# frozen_string_literal: true\n\nrequire \"cgi\"\n\nmodule SampleProject\n  class UserRenderer\n    def self.render_profile(user)\n      <<~HTML\n        <div class=\"profile\">\n          <h1>#{escape(user[:name])}</h1>\n          <p>#{escape(user[:bio])}</p>\n          <a href=\"#{escape(user[:website])}\">Website</a>\n        </div>\n      HTML\n    end\n\n    def self.render_list(users)\n      items = users.map { |u| \"<li>#{escape(u[:name])}</li>\" }.join\n      \"<ul>#{items}</ul>\"\n    end\n\n    def self.escape(text)\n      CGI.escapeHTML(text.to_s)\n    end\n  end\nend\n"
+    },
+    "assistant_messages": [
+      "",
+      "",
+      "",
+      "",
+      "已修复 `lib/user_renderer.rb` 中的 XSS 漏洞。\n\n改动点：\n- 新增 `require \"cgi\"` 和 `escape` 辅助方法，使用 `CGI.escapeHTML` 对 `user[:name]`、`user[:bio]`、`user[:website]` 进行 HTML 实体编码\n- 覆盖了 `render_profile` 和 `render_list` 中所有直接插值用户输入的位置\n\n现在像 `<script>alert(1)</script>` 这样的输入会被安全地转义为 `&lt;script&gt;alert(1)&lt;/script&gt;`，不再能注入恶意脚本。"
+    ],
+    "total_assistant_chars": 311
+  },
+  "git_workflow": {
+    "success": true,
+    "iterations": 8,
+    "total_cost": 0.0,
+    "cost_source": "estimated",
+    "duration_seconds": 31.94,
+    "cache_creation_input_tokens": 0,
+    "cache_read_input_tokens": 48640,
+    "total_requests": 8,
+    "cache_hit_requests": 8,
+    "file_changes": {},
+    "assistant_messages": [
+      "",
+      "",
+      "",
+      "",
+      "",
+      "",
+      "",
+      "修复完成。我在 `lib/user_renderer.rb` 中添加了 `escape_html` 方法，并对 `render_profile` 和 `render_list` 中所有用户输入进行了 HTML 转义，消除了 XSS 漏洞。然后使用 `git add lib/user_renderer.rb` 仅将该文件加入了暂存区。"
+    ],
+    "total_assistant_chars": 167
+  }
+}

data/benchmark/runner.rb

@@ -0,0 +1,441 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Benchmark runner for system prompt A/B testing
+# Usage: ruby benchmark/runner.rb
+
+require "fileutils"
+require "json"
+require "tmpdir"
+
+project_root = File.expand_path("..", __dir__)
+$LOAD_PATH.unshift File.join(project_root, "lib")
+require "clacky"
+
+class BenchmarkRunner
+  PROMPT_FILES = {
+    base: "lib/clacky/default_agents/base_prompt.md",
+    coding: "lib/clacky/default_agents/coding/system_prompt.md",
+    general: "lib/clacky/default_agents/general/system_prompt.md"
+  }.freeze
+
+  FIXTURE_DIR = File.expand_path("fixtures/sample_project", __dir__)
+  RESULTS_DIR = File.expand_path("results", __dir__)
+
+  TASKS = [
+    {
+      name: "simple_edit",
+      description: "Rename methods to snake_case across files",
+      prompt: "Rename the method `calculateTotal` to `calculate_total` in all files. Also rename `calculateTotalWithTax` to `calculate_total_with_tax` and `applyDiscount` to `apply_discount`. Update all references and the test file too.",
+      agent_profile: "coding"
+    },
+    {
+      name: "feature_addition",
+      description: "Add a new /products API endpoint with tests",
+      prompt: "Add a new `/products` endpoint to the ApiHandler that returns products from the store with optional pagination via `page` and `per_page` params. Also create a test file `spec/api_handler_spec.rb` with basic tests for this endpoint.",
+      agent_profile: "coding"
+    },
+    {
+      name: "refactoring",
+      description: "Extract helper method from duplicated pattern",
+      prompt: "In order_calculator.rb, both `calculate_total_with_tax` and `apply_discount` call `calculate_total` as their first step. Refactor to eliminate this duplication in the cleanest way possible. Do not over-engineer.",
+      agent_profile: "coding"
+    },
+    {
+      name: "bug_fix",
+      description: "Fix XSS vulnerability in HTML rendering",
+      prompt: "Fix the XSS vulnerability in user_renderer.rb. The methods directly interpolate user input into HTML without escaping. Make the rendering safe against XSS attacks.",
+      agent_profile: "coding"
+    },
+    {
+      name: "git_workflow",
+      description: "Fix bug and stage changes safely with git",
+      prompt: "Fix the XSS vulnerability in user_renderer.rb, then use git to stage only the changed file for commit. Do NOT stage all files.",
+      agent_profile: "coding"
+    }
+  ].freeze
+
+  def initialize
+    @project_root = File.expand_path("..", __dir__)
+    @original_prompts = read_current_prompts
+    FileUtils.mkdir_p(RESULTS_DIR)
+  end
+
+  def run
+    run_baseline
+    run_treatment
+    run_report
+  end
+
+  def run_baseline
+    puts "=" * 70
+    puts "OpenClacky System Prompt Benchmark - BASELINE"
+    puts "=" * 70
+    puts "Project: #{@project_root}"
+    puts "Model: #{agent_config.model_name}"
+    puts "Tasks: #{TASKS.length}"
+    puts
+
+    unless git_clean?
+      puts "WARNING: Prompt files have uncommitted changes. Baseline may not reflect main."
+      puts
+    end
+
+    baseline_prompts = read_baseline_prompts
+    write_prompts(baseline_prompts)
+    results = run_all_tasks(:baseline)
+    write_results("baseline", results)
+
+    # Restore treatment prompts
+    write_prompts(@original_prompts)
+    puts "\nBaseline complete. Results saved."
+    results
+  rescue => e
+    write_prompts(@original_prompts)
+    raise
+  end
+
+  def run_treatment
+    puts "=" * 70
+    puts "OpenClacky System Prompt Benchmark - TREATMENT"
+    puts "=" * 70
+    puts "Project: #{@project_root}"
+    puts "Model: #{agent_config.model_name}"
+    puts "Tasks: #{TASKS.length}"
+    puts
+
+    # Ensure treatment prompts are active
+    write_prompts(@original_prompts)
+    results = run_all_tasks(:treatment)
+    write_results("treatment", results)
+
+    puts "\nTreatment complete. Results saved."
+    results
+  end
+
+  def run_report
+    baseline_file = Dir.glob(File.join(RESULTS_DIR, "baseline_*.json")).max
+    treatment_file = Dir.glob(File.join(RESULTS_DIR, "treatment_*.json")).max
+
+    unless baseline_file
+      puts "ERROR: No baseline results found in #{RESULTS_DIR}"
+      exit 1
+    end
+    unless treatment_file
+      puts "ERROR: No treatment results found in #{RESULTS_DIR}"
+      exit 1
+    end
+
+    baseline = JSON.parse(File.read(baseline_file), symbolize_names: true)
+    treatment = JSON.parse(File.read(treatment_file), symbolize_names: true)
+
+    puts "=" * 70
+    puts "COMPARISON REPORT"
+    puts "=" * 70
+    puts "Baseline: #{File.basename(baseline_file)}"
+    puts "Treatment: #{File.basename(treatment_file)}"
+    puts
+    compare_and_print(baseline, treatment)
+
+    # Save combined report
+    report_path = File.join(RESULTS_DIR, "report_#{timestamp}.json")
+    File.write(report_path, JSON.pretty_generate({
+      baseline: baseline,
+      treatment: treatment,
+      meta: {
+        model: agent_config.model_name,
+        timestamp: Time.now.iso8601,
+        tasks: TASKS.map { |t| t[:name] }
+      }
+    }))
+    puts
+    puts "Full report saved to: #{report_path}"
+  end
+
+  private
+
+  def agent_config
+    @agent_config ||= Clacky::AgentConfig.load
+  end
+
+  def read_current_prompts
+    prompts = {}
+    PROMPT_FILES.each do |key, rel_path|
+      full_path = File.join(@project_root, rel_path)
+      prompts[key] = File.read(full_path)
+    end
+    prompts
+  end
+
+  def read_baseline_prompts
+    prompts = {}
+    PROMPT_FILES.each do |key, rel_path|
+      content = `git -C "#{@project_root}" show main:"#{rel_path}" 2>/dev/null`
+      if $?.success? && !content.empty?
+        prompts[key] = content
+      else
+        puts "  Warning: Could not read #{rel_path} from main, using current"
+        prompts[key] = @original_prompts[key]
+      end
+    end
+    prompts
+  end
+
+  def write_prompts(prompts)
+    prompts.each do |key, content|
+      rel_path = PROMPT_FILES[key]
+      full_path = File.join(@project_root, rel_path)
+      File.write(full_path, content)
+    end
+  end
+
+  def git_clean?
+    PROMPT_FILES.values.all? do |rel_path|
+      status = `git -C "#{@project_root}" status --porcelain "#{rel_path}" 2>/dev/null`
+      status.strip.empty?
+    end
+  end
+
+  def run_all_tasks(variant)
+    results = {}
+    TASKS.each_with_index do |task, idx|
+      puts
+      puts "[#{idx + 1}/#{TASKS.length}] #{task[:name]}: #{task[:description]}"
+      results[task[:name]] = run_task(task, variant)
+    end
+    results
+  end
+
+  def run_task(task, variant)
+    tmp_dir = File.join(Dir.tmpdir, "clacky_benchmark_#{variant}_#{task[:name]}_#{Process.pid}_#{Time.now.to_i}")
+    FileUtils.cp_r(FIXTURE_DIR, tmp_dir)
+
+    # Ensure tmp_dir is a git repo (cp_r preserves .git)
+    Dir.chdir(tmp_dir) do
+      system("git config user.email 'benchmark@test.com' >/dev/null 2>&1")
+      system("git config user.name 'Benchmark' >/dev/null 2>&1")
+    end
+
+    config = agent_config.dup
+    config.permission_mode = :auto_approve
+
+    client = Clacky::Client.new(
+      config.api_key,
+      base_url: config.base_url,
+      model: config.model_name,
+      anthropic_format: config.anthropic_format?
+    )
+
+    agent = Clacky::Agent.new(
+      client, config,
+      working_dir: tmp_dir,
+      ui: BenchmarkUI.new,
+      profile: task[:agent_profile],
+      session_id: Clacky::SessionManager.generate_id,
+      source: :manual
+    )
+
+    start_time = Time.now
+    agent.run(task[:prompt])
+    duration = Time.now - start_time
+
+    # Collect metrics
+    metrics = {
+      success: true,
+      iterations: agent.iterations,
+      total_cost: agent.total_cost.round(6),
+      cost_source: agent.cost_source.to_s,
+      duration_seconds: duration.round(2),
+      cache_creation_input_tokens: agent.cache_stats[:cache_creation_input_tokens],
+      cache_read_input_tokens: agent.cache_stats[:cache_read_input_tokens],
+      total_requests: agent.cache_stats[:total_requests],
+      cache_hit_requests: agent.cache_stats[:cache_hit_requests]
+    }
+
+    # Collect file changes
+    metrics[:file_changes] = collect_file_changes(tmp_dir)
+
+    # Collect assistant output for qualitative analysis
+    metrics[:assistant_messages] = agent.history.to_a
+      .select { |m| m[:role] == "assistant" }
+      .map { |m| extract_text(m[:content]) }
+      .compact
+
+    metrics[:total_assistant_chars] = metrics[:assistant_messages].join.length
+
+    # Cleanup
+    FileUtils.rm_rf(tmp_dir)
+
+    print_metrics(metrics)
+    metrics
+  rescue => e
+    FileUtils.rm_rf(tmp_dir) if defined?(tmp_dir) && tmp_dir
+    error_result = {
+      success: false,
+      error: e.message,
+      error_class: e.class.name,
+      iterations: defined?(agent) ? agent&.iterations : 0,
+      total_cost: defined?(agent) ? agent&.total_cost&.round(6) : 0
+    }
+    puts "  ERROR: #{e.message}"
+    error_result
+  end
+
+  def collect_file_changes(dir)
+    changes = {}
+    Dir.chdir(dir) do
+      # Get list of modified files
+      modified = `git diff --name-only 2>/dev/null`.strip.split("\n").reject(&:empty?)
+      modified.each do |f|
+        next unless File.exist?(f)
+        changes[f] = File.read(f)
+      end
+    end
+    changes
+  end
+
+  def extract_text(content)
+    case content
+    when String then content
+    when Array
+      text_parts = content.select { |p| p.is_a?(Hash) && p[:type] == "text" }
+      text_parts.map { |p| p[:text] }.join(" ")
+    else
+      nil
+    end
+  end
+
+  def print_metrics(metrics)
+    if metrics[:success]
+      puts "  Iterations: #{metrics[:iterations]} | Cost: $#{metrics[:total_cost]} | Duration: #{metrics[:duration_seconds]}s"
+      puts "  Cache: write=#{metrics[:cache_creation_input_tokens]} read=#{metrics[:cache_read_input_tokens]}"
+      puts "  Assistant chars: #{metrics[:total_assistant_chars]}"
+      puts "  Files changed: #{metrics[:file_changes]&.keys&.join(', ') || 'none'}"
+    else
+      puts "  FAILED: #{metrics[:error]}"
+    end
+  end
+
+  def write_results(name, results)
+    path = File.join(RESULTS_DIR, "#{name}_#{timestamp}.json")
+    File.write(path, JSON.pretty_generate(results))
+    puts "\n#{name.capitalize} results saved to: #{path}"
+  end
+
+  def timestamp
+    @timestamp ||= Time.now.strftime("%Y%m%d_%H%M%S")
+  end
+
+  def compare_and_print(baseline, treatment)
+    puts
+    printf "%-20s %12s %12s %12s\n", "Task", "Baseline", "Treatment", "Delta"
+    puts "-" * 60
+
+    TASKS.each do |task|
+      task_key = task[:name].to_sym
+      b = baseline[task_key] || {}
+      t = treatment[task_key] || {}
+
+      next unless b[:success] && t[:success]
+
+      b_cost = b[:total_cost] || 0
+      t_cost = t[:total_cost] || 0
+      cost_delta = b_cost > 0 ? "#{(t_cost / b_cost * 100).round(1)}%" : "N/A"
+
+      b_iter = b[:iterations] || 0
+      t_iter = t[:iterations] || 0
+
+      b_chars = b[:total_assistant_chars] || 0
+      t_chars = t[:total_assistant_chars] || 0
+      chars_delta = b_chars > 0 ? "#{(t_chars / b_chars.to_f * 100).round(1)}%" : "N/A"
+
+      printf "%-20s\n", task[:name]
+      printf "  Cost:              $%-10.6f $%-10.6f %s\n", b_cost, t_cost, cost_delta
+      printf "  Iterations:        %-11d %-11d %s\n", b_iter, t_iter, "#{t_iter - b_iter > 0 ? '+' : ''}#{t_iter - b_iter}"
+      printf "  Assistant chars:   %-11d %-11d %s\n", b_chars, t_chars, chars_delta
+      puts
+    end
+
+    # Totals
+    b_total_cost = 0
+    t_total_cost = 0
+    b_total_iter = 0
+    t_total_iter = 0
+    b_total_chars = 0
+    t_total_chars = 0
+
+    TASKS.each do |task|
+      task_key = task[:name].to_sym
+      b = baseline[task_key] || {}
+      t = treatment[task_key] || {}
+      next unless b[:success] && t[:success]
+
+      b_total_cost += b[:total_cost] || 0
+      t_total_cost += t[:total_cost] || 0
+      b_total_iter += b[:iterations] || 0
+      t_total_iter += t[:iterations] || 0
+      b_total_chars += b[:total_assistant_chars] || 0
+      t_total_chars += t[:total_assistant_chars] || 0
+    end
+
+    puts "-" * 60
+    printf "%-20s\n", "TOTALS"
+    cost_pct = b_total_cost > 0 ? (t_total_cost / b_total_cost * 100).round(1) : 0
+    printf "  Total cost:        $%-10.6f $%-10.6f %s%%\n", b_total_cost, t_total_cost, cost_pct
+    printf "  Total iterations:  %-11d %-11d %+d\n", b_total_iter, t_total_iter, t_total_iter - b_total_iter
+    chars_pct = b_total_chars > 0 ? (t_total_chars / b_total_chars.to_f * 100).round(1) : 0
+    printf "  Total chars:       %-11d %-11d %s%%\n", b_total_chars, t_total_chars, chars_pct
+  end
+
+  # Minimal UI that captures output without displaying
+  class BenchmarkUI
+    def log(msg, level: :info); end
+    def show_assistant_message(content, files: []); end
+    def show_tool_call(name, args); end
+    def show_tool_result(result); end
+    def show_tool_stdout(lines); end
+    def show_tool_error(error); end
+    def show_tool_args(formatted_args); end
+    def show_file_write_preview(path, is_new_file:); end
+    def show_file_edit_preview(path); end
+    def show_file_error(error_message); end
+    def show_shell_preview(command); end
+    def show_diff(old_content, new_content, max_lines: 50); end
+    def show_token_usage(token_data); end
+    def show_complete(iterations:, cost:, duration: nil, cache_stats: nil, awaiting_user_feedback: false, cost_source: nil); end
+    def append_output(content); end
+    def show_info(message, prefix_newline: true); end
+    def show_warning(message); end
+    def show_error(message); end
+    def show_success(message); end
+    def show_progress(message = nil, prefix_newline: true, progress_type: "thinking", phase: "active", metadata: {}); end
+    def start_progress(message: nil, style: :primary, quiet_on_fast_finish: false); end
+    def with_progress(message: nil, style: :primary, quiet_on_fast_finish: false)
+      yield if block_given?
+    end
+    def update_sessionbar(tasks: nil, cost: nil, cost_source: nil, status: nil, latency: nil); end
+    def update_todos(todos); end
+    def set_working_status; end
+    def set_idle_status; end
+    def request_confirmation(message, default: true); end
+    def clear_input; end
+    def set_input_tips(message, type: :info); end
+    def stop; end
+  end
+end
+
+if __FILE__ == $0
+  variant = ARGV[0]&.downcase
+  runner = BenchmarkRunner.new
+
+  case variant
+  when "baseline"
+    runner.run_baseline
+  when "treatment"
+    runner.run_treatment
+  when "report"
+    runner.run_report
+  else
+    runner.run
+  end
+end