openclacky 1.3.4 → 1.3.6

data/lib/clacky/cli.rb

@@ -654,7 +654,6 @@ module Clacky
 
       # Handle agent error/interrupt with cleanup
       def handle_agent_exception(ui_controller, agent, session_manager, exception)
-        Clacky::Logger.warn("[ph_debug] handle_agent_exception", klass: exception.class.name, msg: exception.message.to_s[0, 200])
         ui_controller.show_progress(phase: "done")
         ui_controller.set_idle_status
 

data/lib/clacky/client.rb

@@ -34,11 +34,6 @@ module Clacky
       # some OpenRouter-compatible relays only honour Bearer — send both).
       @provider_id = provider_id
 
-      # Determine vision support once at construction time.
-      # Non-vision models (DeepSeek, Kimi, MiniMax, etc.) reject image_url
-      # content blocks; the conversion layer strips them when this is false.
-      @vision_supported = Providers.supports?(provider_id, :vision, model_name: @model)
-
       # Optional override for Faraday read_timeout (e.g. benchmark calls).
       # nil means use the default (300s for streaming).
       @read_timeout = read_timeout
@@ -343,9 +338,12 @@ module Clacky
       # OpenRouter proxies Claude with the same cache_control field convention as Anthropic direct.
       messages = apply_message_caching(messages) if caching_enabled
 
+      # Vision support is resolved against the request's actual model (which may
+      # differ from @model after a runtime switch or fallback override), so the
+      # conversion layer strips image_url blocks for non-vision models.
       body = MessageFormat::OpenAI.build_request_body(
         messages, model, tools, max_tokens, caching_enabled,
-        vision_supported: @vision_supported,
+        vision_supported: Providers.supports?(@provider_id, :vision, model_name: model),
         reasoning_effort: reasoning_effort
       )
       return send_openai_stream_request(body, on_chunk) if on_chunk
@@ -641,35 +639,34 @@ module Clacky
         raise InsufficientCreditError.new(
           "[LLM] #{I18n.t("llm.error.insufficient_credit")}",
           error_code: "insufficient_credit",
-          provider_id: @provider_id
+          provider_id: @provider_id,
+          raw_message: error_message
         )
       end
 
       case response.status
       when 400
-        # Well-behaved APIs (Anthropic, OpenAI) never put quota/availability issues in 400.
-        # However, some proxy/relay providers do — so we inspect the message first.
-        # Also, Bedrock returns ThrottlingException as 400 instead of 429.
         if error_message.match?(/ThrottlingException|unavailable|quota/i)
           raise RetryableError, "[LLM] #{I18n.t("llm.error.rate_limit_400")}"
         end
 
-        # True bad request — our message was malformed. Roll back history so the
-        # broken message is not replayed on the next user turn.
         raise BadRequestError.new(
           "[LLM] Client request error: #{error_message}",
-          display_message: "[LLM] #{I18n.t("llm.error.bad_request")}"
+          display_message: "[LLM] #{I18n.t("llm.error.bad_request")}",
+          raw_message: error_message
         )
-      when 401 then raise AgentError, "[LLM] #{I18n.t("llm.error.invalid_api_key")}"
+      when 401
+        raise AgentError.new("[LLM] #{I18n.t("llm.error.invalid_api_key")}", raw_message: error_message)
       when 403
         i18n_key = "llm.error.403.#{error_code}"
         translated = I18n.t(i18n_key)
         translated = I18n.t("llm.error.403.default") if translated == i18n_key
-        raise AgentError, "[LLM] #{translated}"
-      when 404 then raise AgentError, "[LLM] #{I18n.t("llm.error.endpoint_not_found")}"
+        raise AgentError.new("[LLM] #{translated}", raw_message: error_message)
+      when 404
+        raise AgentError.new("[LLM] #{I18n.t("llm.error.endpoint_not_found")}", raw_message: error_message)
       when 429 then raise RetryableError, "[LLM] #{I18n.t("llm.error.rate_limit_429")}"
       when 500..599 then raise RetryableError, "[LLM] #{I18n.t("llm.error.server_error", status: response.status)}"
-      else raise AgentError, "[LLM] #{I18n.t("llm.error.unexpected", status: response.status)}"
+      else raise AgentError.new("[LLM] #{I18n.t("llm.error.unexpected", status: response.status)}", raw_message: error_message)
       end
     end
 

data/lib/clacky/default_agents/_panels/time_machine/panel.js

@@ -19,6 +19,25 @@
 (() => {
   if (!window.Clacky || !Clacky.ext) return;
 
+  // The currently mounted panel's state, refreshed on every mount. A single WS
+  // hook (registered once below) reloads it when the active session completes a
+  // task, so new snapshots appear without a manual refresh. Kept as a closure
+  // singleton because WS.onEvent has no unsubscribe and the panel re-mounts on
+  // each session switch.
+  let _activeState = null;
+  let _wsHooked = false;
+
+  function _hookWs() {
+    if (_wsHooked || typeof WS === "undefined") return;
+    _wsHooked = true;
+    WS.onEvent((ev) => {
+      if (ev && ev.type === "complete" && _activeState &&
+          ev.session_id === _activeState.sessionId) {
+        loadHistory(_activeState);
+      }
+    });
+  }
+
   const t = (k, fallback) => {
     const v = (typeof I18n !== "undefined") ? I18n.t(k) : null;
     return (v && v !== k) ? v : fallback;
@@ -630,6 +649,9 @@
     d.mask.onclick = onMaskClick;
     document.addEventListener("keydown", onKey);
 
+    _activeState = state;
+    _hookWs();
+
     loadHistory(state);
     return root;
   }, {

data/lib/clacky/default_agents/base_prompt.md

@@ -9,6 +9,7 @@
 
 - **ALWAYS use `glob` tool to find files — NEVER use shell `find` command for file discovery**
 - **All operations default to the working directory** (shown in session context)
+- **NEVER write tool calls as text** (e.g. `<invoke name=...>`, `<function_calls>`). Use the structured tool_calls field — text won't execute.
 
 ## Response Style
 

data/lib/clacky/default_extensions/meeting/handler.rb

@@ -0,0 +1,331 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+require "base64"
+require "tmpdir"
+
+# Meeting Extension — real-time transcription, on-demand agent Q&A, and
+# post-meeting summarization. Mounted at /api/ext/meeting/.
+class MeetingExt < Clacky::ApiExtension
+  timeout 30
+
+  MEETINGS_ROOT = File.expand_path("~/.clacky/meetings")
+  VOCABULARY_PATH = File.join(MEETINGS_ROOT, "vocabulary.txt")
+  DEFAULT_VOCABULARY = "Clacky, OpenClacky, openclacky"
+
+  # annotate is a read-only analysis: block every side-effecting tool so the
+  # forked subagent can only read/think, never write files, run commands,
+  # spawn more work, or prompt the user.
+  WRITE_TOOLS = %w[write edit terminal trash_manager invoke_skill request_user_feedback browser].freeze
+
+  # ── Vocabulary (STT biasing hints) ────────────────────────────────────────
+
+  # GET /api/ext/meeting/vocabulary
+  get "/vocabulary" do
+    json(vocabulary: read_vocabulary)
+  end
+
+  # POST /api/ext/meeting/vocabulary
+  # body: { vocabulary }
+  post "/vocabulary" do
+    text = json_body["vocabulary"].to_s.strip
+    FileUtils.mkdir_p(MEETINGS_ROOT)
+    File.write(VOCABULARY_PATH, text)
+    json(ok: true, vocabulary: text)
+  end
+
+  # ── Lifecycle ─────────────────────────────────────────────────────────────
+
+  # POST /api/ext/meeting/start
+  # body: { session_id }
+  # Creates a new meeting tied to the current session.
+  post "/start" do
+    sid = json_body["session_id"]
+    error!("session_id required", status: 422) unless sid && !sid.empty?
+
+    meeting_id = "mtg-#{Time.now.strftime('%Y%m%d-%H%M%S')}-#{SecureRandom.hex(4)}"
+    dir = File.join(MEETINGS_ROOT, sid, meeting_id)
+    FileUtils.mkdir_p(dir)
+
+    meta = { session_id: sid, meeting_id: meeting_id, started_at: Time.now.utc.iso8601 }
+    File.write(File.join(dir, "meta.json"), JSON.pretty_generate(meta))
+    File.write(File.join(dir, "transcript.jsonl"), "")
+
+    json(meeting_id: meeting_id, dir: dir)
+  end
+
+  # POST /api/ext/meeting/end
+  # body: { session_id, meeting_id }
+  # Finalizes the meeting and triggers summarization via the session agent.
+  post "/end" do
+    sid, mid = json_body.values_at("session_id", "meeting_id")
+    error!("session_id and meeting_id required", status: 422) unless sid && mid
+
+    dir = meeting_dir(sid, mid)
+    error!("meeting not found", status: 404) unless File.directory?(dir)
+
+    meta_path = File.join(dir, "meta.json")
+    meta = JSON.parse(File.read(meta_path))
+    meta["ended_at"] = Time.now.utc.iso8601
+    File.write(meta_path, JSON.pretty_generate(meta))
+
+    transcript_path = File.join(dir, "transcript.jsonl")
+    lines = File.readlines(transcript_path).map { |l| JSON.parse(l)["text"] }.reject(&:empty?)
+    transcript = lines.join("\n")
+
+    logger.info("end: sid=#{sid} mid=#{mid} lines=#{lines.size} transcript_len=#{transcript.length}")
+
+    if transcript.strip.empty?
+      logger.warn("end: transcript is empty, skipping summarization")
+      json(ok: true, meeting_id: mid, skipped: true)
+      next
+    end
+
+    prompt = <<~PROMPT
+      A meeting just ended. Invoke the "meeting-summarizer" skill to generate the meeting minutes from the transcript below.
+
+      Transcript:
+      #{transcript}
+    PROMPT
+
+    begin
+      submit_task(sid, prompt, display_message: "🛑 Meeting ended — generating meeting minutes…")
+      logger.info("end: submit_task succeeded sid=#{sid}")
+    rescue => e
+      logger.error("end: submit_task failed sid=#{sid} error=#{e.message}")
+      json(ok: false, meeting_id: mid, error: e.message)
+      next
+    end
+
+    json(ok: true, meeting_id: mid)
+  end
+
+  # ── Transcription ─────────────────────────────────────────────────────────
+
+  # POST /api/ext/meeting/transcribe
+  # body: { session_id, meeting_id, audio_base64, format: "wav" }
+  # Sends audio chunk to LLM proxy for STT, appends result to transcript.
+  post "/transcribe" do
+    sid, mid = json_body.values_at("session_id", "meeting_id")
+    audio_b64 = json_body["audio_base64"]
+    error!("session_id, meeting_id, audio_base64 required", status: 422) unless sid && mid && audio_b64
+
+    dir = meeting_dir(sid, mid)
+    error!("meeting not found", status: 404) unless File.directory?(dir)
+
+    mime = json_body["mime_type"].to_s.split(";").first.strip
+    mime = "audio/webm" if mime.empty?
+    vocabulary = json_body["vocabulary"].to_s.strip
+    result = call_stt(audio_b64, mime, vocabulary)
+
+    if result["success"]
+      text = result["text"].to_s.strip
+      unless text.empty?
+        entry = { ts: Time.now.utc.iso8601, text: text }
+        File.open(File.join(dir, "transcript.jsonl"), "a") { |f| f.puts(JSON.generate(entry)) }
+      end
+      json(text: text)
+    else
+      error!(result["error"] || "STT failed", status: 502)
+    end
+  end
+
+  # ── Agent Q&A (when @-mentioned) ─────────────────────────────────────────
+
+  # POST /api/ext/meeting/ask
+  # body: { session_id, meeting_id, question }
+  # Submits a question to the session agent with recent transcript as context.
+  post "/ask" do
+    sid, mid = json_body.values_at("session_id", "meeting_id")
+    question = json_body["question"]
+    error!("session_id, meeting_id, question required", status: 422) unless sid && mid && question
+
+    dir = meeting_dir(sid, mid)
+    error!("meeting not found", status: 404) unless File.directory?(dir)
+
+    context = recent_transcript(dir, minutes: 5)
+
+    prompt = <<~PROMPT
+      [Meeting Mode] You are in a team meeting and have been called on to speak. Based on the recent transcript below, answer the question concisely.
+      Keep it short — one or two sentences. Do not elaborate at length.
+
+      --- Recent Transcript ---
+      #{context}
+
+      --- Question ---
+      #{question}
+    PROMPT
+
+    submit_task(sid, prompt, display_message: "🎤 #{question}")
+    json(ok: true)
+  end
+
+  # ── Annotation (periodic background tagging) ──────────────────────────────
+
+  # POST /api/ext/meeting/annotate
+  # body: { session_id, meeting_id }
+  # Analyzes recent transcript and returns tags (decisions, actions, AI-answerable).
+  # Runs as a one-off side LLM call — it must NOT enter the session, otherwise
+  # its raw JSON would pollute the chat transcript.
+  post "/annotate" do
+    sid, mid = json_body.values_at("session_id", "meeting_id")
+    error!("session_id and meeting_id required", status: 422) unless sid && mid
+
+    dir = meeting_dir(sid, mid)
+    error!("meeting not found", status: 404) unless File.directory?(dir)
+
+    context = recent_transcript(dir, minutes: 2)
+    next json(annotations: []) if context.strip.empty?
+
+    prompt = <<~PROMPT
+      Analyze the following meeting transcript excerpt and identify:
+      1. Decisions (something someone decided)
+      2. Action Items (a task assigned to someone)
+      3. AI-answerable questions (technical/factual questions asked but not yet answered)
+
+      Output a JSON array only, no prose, no code fences. Each item:
+      {"type":"decision|action|question","text":"...","speaker":"..."}
+      If none found, output [].
+
+      Transcript:
+      #{context}
+    PROMPT
+
+    result = dispatch_to_session(sid, prompt, model: "lite", forbidden_tools: WRITE_TOOLS)
+    next json(annotations: [], busy: true) if result[:busy]
+
+    json(annotations: parse_annotations(result[:text].to_s))
+  rescue Clacky::ApiExtension::Halt
+    raise
+  rescue StandardError => e
+    logger.error("annotate failed: #{e.message}")
+    json(annotations: [])
+  end
+
+  # ── Transcript retrieval ──────────────────────────────────────────────────
+
+  # GET /api/ext/meeting/transcript/:session_id/:meeting_id
+  get "/transcript/:session_id/:meeting_id" do
+    sid = params[:session_id]
+    mid = params[:meeting_id]
+    dir = meeting_dir(sid, mid)
+    error!("meeting not found", status: 404) unless File.directory?(dir)
+
+    path = File.join(dir, "transcript.jsonl")
+    lines = File.exist?(path) ? File.readlines(path).map { |l| JSON.parse(l) } : []
+    json(transcript: lines)
+  end
+
+  # GET /api/ext/meeting/active/:session_id
+  # Returns the most recent in-progress meeting (no ended_at) for the session,
+  # so a page refresh can restore the live captions instead of losing them.
+  get "/active/:session_id" do
+    sid = params[:session_id]
+    session_root = File.join(MEETINGS_ROOT, sid)
+    next json(active: false) unless File.directory?(session_root)
+
+    dir = active_meeting_dir(session_root)
+    next json(active: false) unless dir
+
+    mid = File.basename(dir)
+    path = File.join(dir, "transcript.jsonl")
+    lines = File.exist?(path) ? File.readlines(path).map { |l| JSON.parse(l) } : []
+    json(active: true, meeting_id: mid, transcript: lines)
+  end
+
+  # POST /api/ext/meeting/speak
+  # body: { text, voice? }
+  # Synthesizes speech from text and returns it as base64 for the browser to play.
+  post "/speak" do
+    text = json_body["text"].to_s.strip
+    error!("text required", status: 422) if text.empty?
+
+    voice = json_body["voice"].to_s.strip
+    voice = nil if voice.empty?
+
+    Dir.mktmpdir("meeting-tts") do |tmp|
+      result = Clacky::Media::Generator.new(agent_config).generate_speech(
+        input: text,
+        voice: voice,
+        output_dir: tmp
+      )
+
+      error!(result["error"] || "TTS failed", status: 502) unless result["success"]
+
+      path = result["audio"]
+      error!("TTS produced no audio", status: 502) unless path && File.exist?(path)
+
+      audio_b64 = Base64.strict_encode64(File.binread(path))
+      mime = result["mime_type"] || "audio/wav"
+      json(audio_base64: audio_b64, mime_type: mime)
+    end
+  rescue Clacky::ApiExtension::Halt
+    raise
+  rescue StandardError => e
+    logger.error("TTS call failed: #{e.message}")
+    error!(e.message, status: 502)
+  end
+
+  private def meeting_dir(session_id, meeting_id)
+    File.join(MEETINGS_ROOT, session_id, meeting_id)
+  end
+
+  private def active_meeting_dir(session_root)
+    Dir.children(session_root)
+       .map { |name| File.join(session_root, name) }
+       .select { |d| File.directory?(d) && File.exist?(File.join(d, "meta.json")) }
+       .reject { |d| (JSON.parse(File.read(File.join(d, "meta.json"))) rescue {})["ended_at"] }
+       .max_by { |d| File.mtime(File.join(d, "meta.json")) }
+  end
+
+  private def read_vocabulary
+    return DEFAULT_VOCABULARY unless File.exist?(VOCABULARY_PATH)
+
+    saved = File.read(VOCABULARY_PATH).strip
+    saved.empty? ? "" : saved
+  end
+
+  private def recent_transcript(dir, minutes: 5)
+    path = File.join(dir, "transcript.jsonl")
+    return "" unless File.exist?(path)
+
+    cutoff = Time.now.utc - (minutes * 60)
+    File.readlines(path).filter_map do |line|
+      entry = JSON.parse(line)
+      ts = Time.parse(entry["ts"]) rescue Time.at(0)
+      entry["text"] if ts >= cutoff
+    end.join("\n")
+  end
+
+  private def parse_annotations(reply)
+    json = reply.strip
+    json = json.sub(/\A```(?:json)?\s*/, "").sub(/\s*```\z/, "") # strip code fences if any
+    start = json.index("[")
+    finish = json.rindex("]")
+    return [] unless start && finish && finish > start
+
+    arr = JSON.parse(json[start..finish])
+    return [] unless arr.is_a?(Array)
+
+    arr.filter_map do |item|
+      next unless item.is_a?(Hash)
+      text = item["text"].to_s.strip
+      next if text.empty?
+      { "type" => item["type"].to_s, "text" => text, "speaker" => item["speaker"].to_s }
+    end
+  rescue JSON::ParserError
+    []
+  end
+
+  private def call_stt(audio_base64, mime_type, vocabulary = nil)
+    Clacky::Media::Generator.new(agent_config).generate_transcription(
+      audio_base64: audio_base64,
+      mime_type: mime_type,
+      prompt: vocabulary.to_s.empty? ? nil : vocabulary
+    )
+  rescue StandardError => e
+    logger.error("STT call failed: #{e.message}")
+    { "success" => false, "text" => nil, "error" => e.message }
+  end
+end