openclacky 1.0.0 → 1.0.2

data/lib/clacky/server/session_registry.rb

@@ -218,31 +218,62 @@ module Clacky
 
         ordered = pinned_section + non_pinned
 
-        ordered.map do |s|
-          id = s[:session_id]
-          ls = live[id]
-          {
-            id:            id,
-            name:          ls&.dig(:name) || s[:name] || "",
-            status:        ls ? ls[:status].to_s : "idle",
-            error:         ls ? ls[:error] : nil,
-            model:         ls&.dig(:model),
-            source:        s_source(s),
-            agent_profile: (s[:agent_profile] || "general").to_s,
-            working_dir:   s[:working_dir],
-            created_at:    s[:created_at],
-            updated_at:    s[:updated_at],
-            total_tasks:   ls&.dig(:total_tasks) || s.dig(:stats, :total_tasks) || 0,
-            total_cost:    ls&.dig(:total_cost)  || s.dig(:stats, :total_cost_usd) || 0.0,
-            cost_source:   (ls&.dig(:cost_source) || s.dig(:stats, :cost_source) || "estimated").to_s,
-            # latest_latency is in-memory only (live sessions) — not persisted
-            # at the session-level on disk. The on-disk source of truth is
-            # per-assistant-message `latency` fields in messages[]. Reloaded
-            # sessions start with nil and get populated on the next LLM call.
-            latest_latency: ls&.dig(:latest_latency),
-            pinned:        s[:pinned] || false,
-          }
+        ordered.map { |s| build_enriched_row(s, live[s[:session_id]]) }
+      end
+
+      # Return the same enriched hash that a `list` row would produce, for a
+      # single session — merging on-disk fields with in-memory live fields.
+      # Returns nil if the session is unknown on disk.
+      #
+      # This is the targeted, O(1) counterpart to `list` used by the WS layer
+      # when it only needs one row (e.g. pushing a fresh snapshot to a client
+      # that just (re)subscribed, or broadcasting a status-change update).
+      def snapshot(session_id)
+        return nil unless @session_manager
+        disk = @session_manager.load(session_id)
+        return nil unless disk
+
+        live = @mutex.synchronize do
+          s = @sessions[session_id]
+          next nil unless s
+          model_info = s[:agent]&.current_model_info
+          live_name  = s[:agent]&.name
+          live_name  = nil if live_name&.empty?
+          { status: s[:status], error: s[:error], model: model_info&.dig(:model),
+            name: live_name, total_tasks: s[:agent]&.total_tasks,
+            total_cost: s[:agent]&.total_cost, cost_source: s[:agent]&.cost_source,
+            latest_latency: s[:agent]&.latest_latency }
         end
+
+        build_enriched_row(disk, live)
+      end
+
+      # Merge a single disk-side session hash with the corresponding live
+      # in-memory agent fields (may be nil) into the row shape the frontend
+      # consumes.
+      private def build_enriched_row(s, ls)
+        id = s[:session_id]
+        {
+          id:            id,
+          name:          ls&.dig(:name) || s[:name] || "",
+          status:        ls ? ls[:status].to_s : "idle",
+          error:         ls ? ls[:error] : nil,
+          model:         ls&.dig(:model),
+          source:        s_source(s),
+          agent_profile: (s[:agent_profile] || "general").to_s,
+          working_dir:   s[:working_dir],
+          created_at:    s[:created_at],
+          updated_at:    s[:updated_at],
+          total_tasks:   ls&.dig(:total_tasks) || s.dig(:stats, :total_tasks) || 0,
+          total_cost:    ls&.dig(:total_cost)  || s.dig(:stats, :total_cost_usd) || 0.0,
+          cost_source:   (ls&.dig(:cost_source) || s.dig(:stats, :cost_source) || "estimated").to_s,
+          # latest_latency is in-memory only (live sessions) — not persisted
+          # at the session-level on disk. The on-disk source of truth is
+          # per-assistant-message `latency` fields in messages[]. Reloaded
+          # sessions start with nil and get populated on the next LLM call.
+          latest_latency: ls&.dig(:latest_latency),
+          pinned:        s[:pinned] || false,
+        }
       end
 
 

data/lib/clacky/skill.rb

@@ -514,19 +514,20 @@ module Clacky
             @warnings << "Invalid name '#{@name}' in metadata; using directory name '#{dir_slug}' instead."
             @name = dir_slug
           else
-            # Unrecoverable: both name and directory slug are invalid — mark skill as invalid
-            @invalid        = true
-            @invalid_reason = "Invalid skill name '#{@name}' and directory name '#{dir_slug}' is also not a valid slug. " \
-                              "Expected lowercase letters, numbers, and hyphens (e.g. 'my-skill')."
-            @name = nil
+            # Both name and directory slug are invalid (e.g. contains dots from version suffix).
+            # Record a warning but keep the skill usable — do not mark as invalid.
+            @warnings << "Invalid skill name '#{@name}' and directory name '#{dir_slug}' is also not a valid slug. " \
+                         "Expected lowercase letters, numbers, and hyphens (e.g. 'my-skill')."
+            @name = dir_slug
           end
         end
       else
-        # No name in frontmatter — check the directory slug itself
+        # No name in frontmatter — check the directory slug itself.
+        # Non-conforming names (e.g. version-suffixed dirs like "test-runner-1.0.0")
+        # are allowed with a warning rather than being rejected outright.
         unless valid_slug.call(dir_slug)
-          @invalid        = true
-          @invalid_reason = "Directory name '#{dir_slug}' is not a valid skill slug. " \
-                            "Expected lowercase letters, numbers, and hyphens (e.g. 'my-skill')."
+          @warnings << "Directory name '#{dir_slug}' is not a valid skill slug. " \
+                       "Expected lowercase letters, numbers, and hyphens (e.g. 'my-skill')."
         end
       end
 

data/lib/clacky/skill_loader.rb

@@ -287,20 +287,32 @@ module Clacky
     def load_skills_from_directory(dir, source_type)
       return [] unless dir.exist?
 
+      source_path = case source_type
+      when :global_clacky
+        Pathname.new(ENV.fetch("HOME", "~")).join(".clacky")
+      when :project_clacky
+        Pathname.new(@working_dir)
+      else
+        dir
+      end
+
       skills = []
-      dir.children.select(&:directory?).each do |skill_dir|
-        source_path = case source_type
-        when :global_clacky
-          Pathname.new(ENV.fetch("HOME", "~")).join(".clacky")
-        when :project_clacky
-          Pathname.new(@working_dir)
+      dir.children.select(&:directory?).each do |entry|
+        if entry.join("SKILL.md").exist?
+          # Direct skill directory
+          skill = load_single_skill(entry, source_path, entry.basename.to_s, source_type)
+          skills << skill if skill
         else
-          skill_dir
+          # Treat as a category directory — scan one level deeper for skills.
+          # This allows grouping skills under ~/.clacky/skills/<category>/<skill>/SKILL.md
+          # (e.g. openclaw-imports/my-skill/SKILL.md) without changing the loader contract.
+          entry.children.select(&:directory?).each do |skill_dir|
+            next unless skill_dir.join("SKILL.md").exist?
+
+            skill = load_single_skill(skill_dir, source_path, skill_dir.basename.to_s, source_type)
+            skills << skill if skill
+          end
         end
-
-        skill_name = skill_dir.basename.to_s
-        skill = load_single_skill(skill_dir, source_path, skill_name, source_type)
-        skills << skill if skill
       end
       skills
     end

data/lib/clacky/tools/file_reader.rb

@@ -7,7 +7,7 @@ module Clacky
   module Tools
     class FileReader < Base
       self.tool_name = "file_reader"
-      self.tool_description = "Read contents of a file from the filesystem. Supports text files, images (PNG/JPG/GIF/WEBP), and documents (PDF/DOCX/XLSX/PPTX)."
+      self.tool_description = "Read contents of a file from the filesystem. Supports text files, images (PNG/JPG/GIF/WEBP), and documents (PDF/DOCX/XLSX/PPTX — auto-converted to text via parsers, with OCR fallback for scanned PDFs)."
       self.tool_category = "file_system"
       self.tool_parameters = {
         type: "object",
@@ -39,7 +39,7 @@ module Clacky
       MAX_TEXT_FILE_SIZE = 1 * 1024 * 1024
 
       # Maximum content size to return (~10,000 tokens = ~40,000 characters)
-      MAX_CONTENT_CHARS = 80_000
+      MAX_CONTENT_CHARS = 60_000
 
       # Maximum characters per line (prevent single huge lines from bloating tokens)
       MAX_LINE_CHARS = 1000
@@ -70,103 +70,178 @@ module Clacky
         end
 
         begin
-          # Check if file is binary
-          if Utils::FileProcessor.binary_file_path?(expanded_path)
-            return handle_binary_file(expanded_path)
-          end
+          # Delegate to FileProcessor for file type dispatch. FileProcessor is
+          # the single source of truth for how a file becomes a readable form
+          # (parser-extracted text, image base64, archive listing, plain text).
+          # FileReader here only shapes the result for the LLM.
+          ref = Utils::FileProcessor.process_path(expanded_path)
+
+          case ref.type
+          when :image
+            # Images go to LLM as base64 via the image_inject sidecar channel.
+            return handle_image_file(expanded_path)
+
+          when :pdf, :document, :spreadsheet, :presentation
+            # Parser-backed document formats. FileProcessor has already
+            # produced a preview markdown file (or set parse_error on failure).
+            if ref.preview_path && File.exist?(ref.preview_path)
+              return read_text_file(
+                expanded_path,
+                max_lines: max_lines,
+                start_line: start_line,
+                end_line: end_line,
+                source_path: ref.preview_path,
+                parsed_from: ref.type
+              )
+            else
+              return build_parser_failure_result(expanded_path, ref)
+            end
 
-          # Check text file size (only for non-binary files)
-          file_size = File.size(expanded_path)
-          if file_size > MAX_TEXT_FILE_SIZE
-            return {
-              path: expanded_path,
-              content: nil,
-              size_bytes: file_size,
-              error: "Text file too large: #{format_file_size(file_size)} (max: #{format_file_size(MAX_TEXT_FILE_SIZE)}). Please use grep tool to search within this file instead."
-            }
-          end
+          when :text, :csv, :zip
+            # FileProcessor already produced a preview (raw text copy for
+            # text/csv, archive listing for zip/tar). Read the preview with
+            # normal line-range + truncation rules.
+            source = (ref.preview_path && File.exist?(ref.preview_path)) ? ref.preview_path : expanded_path
+            return read_text_file(
+              expanded_path,
+              max_lines: max_lines,
+              start_line: start_line,
+              end_line: end_line,
+              source_path: source
+            )
 
-          # Read text file with optional line range.
-          # Scrub invalid UTF-8 bytes (e.g. GBK-encoded files) so downstream
-          # JSON.generate / history persistence won't blow up later.
-          all_lines = File.readlines(expanded_path).map! { |line| safe_utf8(line) }
-          total_lines = all_lines.size
-
-          # Calculate start index (convert 1-indexed to 0-indexed)
-          start_idx = start_line ? [start_line - 1, 0].max : 0
-
-          # Calculate end index based on parameters
-          if end_line
-            # User specified end_line directly
-            end_idx = [end_line - 1, total_lines - 1].min
-          elsif start_line
-            # start_line + max_lines - 1 (relative to start_line, inclusive)
-            calculated_end_line = start_line + max_lines - 1
-            end_idx = [calculated_end_line - 1, total_lines - 1].min
           else
-            # Read from beginning with max_lines limit
-            end_idx = [max_lines - 1, total_lines - 1].min
-          end
-
-          # Check if start_line exceeds file length first
-          if start_idx >= total_lines
-            return {
-              path: expanded_path,
-              content: nil,
-              lines_read: 0,
-              error: "Invalid line range: start_line #{start_line} exceeds total lines (#{total_lines})"
-            }
-          end
-
-          # Validate range
-          if start_idx > end_idx
-            return {
-              path: expanded_path,
-              content: nil,
-              lines_read: 0,
-              error: "Invalid line range: start_line #{start_line} > end_line #{end_line || (start_line + max_lines)}"
-            }
-          end
-
-          lines = all_lines[start_idx..end_idx] || []
-
-          # Truncate individual lines that are too long
-          lines = lines.map do |line|
-            if line.length > MAX_LINE_CHARS
-              line[0...MAX_LINE_CHARS] + "... [Line truncated - #{line.length} chars]\n"
-            else
-              line
+            # Unknown / :file — could be an unrecognised source file, a binary
+            # blob, or anything else. Fall back to:
+            #   1. If FileProcessor.binary_file_path? says it's binary → report unsupported.
+            #   2. Otherwise → read as plain text (covers .rb, .py, .js, .log, etc.).
+            if Utils::FileProcessor.binary_file_path?(expanded_path)
+              return handle_unsupported_binary(expanded_path, ref)
             end
-          end
-
-          content = lines.join
-          truncated = end_idx < (total_lines - 1)
 
-          # Truncate total content if it exceeds maximum size
-          if content.length > MAX_CONTENT_CHARS
-            content = content[0...MAX_CONTENT_CHARS] +
-                     "\n\n[Content truncated - exceeded #{MAX_CONTENT_CHARS} characters (~10,000 tokens)]" +
-                     "\nUse start_line/end_line parameters to read specific sections, or grep tool to search for keywords."
-            truncated = true
+            return read_text_file(
+              expanded_path,
+              max_lines: max_lines,
+              start_line: start_line,
+              end_line: end_line
+            )
           end
-
+        rescue StandardError => e
           {
             path: expanded_path,
-            content: content,
-            lines_read: lines.size,
-            total_lines: total_lines,
-            truncated: truncated,
+            content: nil,
+            error: "Error reading file: #{e.message}"
+          }
+        end
+      end
+
+      # Read a plain-text file with line-range selection and token-budget
+      # truncation. The source of the text can be:
+      #   - the original file itself (source_path == expanded_path)
+      #   - a parser-generated preview.md for documents (source_path = ref.preview_path)
+      # The reported `path` is always the original file so the LLM sees a
+      # consistent identity.
+      private def read_text_file(display_path, max_lines:, start_line:, end_line:, source_path: nil, parsed_from: nil)
+        source_path ||= display_path
+
+        file_size = File.size(source_path)
+        if file_size > MAX_TEXT_FILE_SIZE
+          return {
+            path: display_path,
+            content: nil,
+            size_bytes: file_size,
+            error: "Text file too large: #{format_file_size(file_size)} (max: #{format_file_size(MAX_TEXT_FILE_SIZE)}). Please use grep tool to search within this file instead."
+          }
+        end
+
+        # Read text file with optional line range.
+        # Scrub invalid UTF-8 bytes (e.g. GBK-encoded files) so downstream
+        # JSON.generate / history persistence won't blow up later.
+        all_lines = File.readlines(source_path).map! { |line| safe_utf8(line) }
+        total_lines = all_lines.size
+
+        # Calculate start index (convert 1-indexed to 0-indexed)
+        start_idx = start_line ? [start_line - 1, 0].max : 0
+
+        # Calculate end index based on parameters
+        if end_line
+          end_idx = [end_line - 1, total_lines - 1].min
+        elsif start_line
+          calculated_end_line = start_line + max_lines - 1
+          end_idx = [calculated_end_line - 1, total_lines - 1].min
+        else
+          end_idx = [max_lines - 1, total_lines - 1].min
+        end
+
+        if total_lines == 0
+          return {
+            path: display_path,
+            content: "",
+            lines_read: 0,
+            total_lines: 0,
+            truncated: false,
             start_line: start_line,
             end_line: end_line,
+            parsed_from: parsed_from&.to_s,
+            source_path: (source_path != display_path ? source_path : nil),
             error: nil
           }
-        rescue StandardError => e
-          {
-            path: expanded_path,
+        end
+
+        # Check if start_line exceeds file length first
+        if start_idx >= total_lines
+          return {
+            path: display_path,
             content: nil,
-            error: "Error reading file: #{e.message}"
+            lines_read: 0,
+            error: "Invalid line range: start_line #{start_line} exceeds total lines (#{total_lines})"
+          }
+        end
+
+        # Validate range
+        if start_idx > end_idx
+          return {
+            path: display_path,
+            content: nil,
+            lines_read: 0,
+            error: "Invalid line range: start_line #{start_line} > end_line #{end_line || (start_line + max_lines)}"
           }
         end
+
+        lines = all_lines[start_idx..end_idx] || []
+
+        # Truncate individual lines that are too long
+        lines = lines.map do |line|
+          if line.length > MAX_LINE_CHARS
+            line[0...MAX_LINE_CHARS] + "... [Line truncated - #{line.length} chars]\n"
+          else
+            line
+          end
+        end
+
+        content = lines.join
+        truncated = end_idx < (total_lines - 1)
+
+        # Truncate total content if it exceeds maximum size
+        if content.length > MAX_CONTENT_CHARS
+          content = content[0...MAX_CONTENT_CHARS] +
+                   "\n\n[Content truncated - exceeded #{MAX_CONTENT_CHARS} characters (~10,000 tokens)]" +
+                   "\nUse start_line/end_line parameters to read specific sections, or grep tool to search for keywords."
+          truncated = true
+        end
+
+        {
+          path: display_path,
+          content: content,
+          lines_read: lines.size,
+          total_lines: total_lines,
+          truncated: truncated,
+          start_line: start_line,
+          end_line: end_line,
+          parsed_from: parsed_from&.to_s,
+          source_path: (source_path != display_path ? source_path : nil),
+          error: nil
+        }
       end
 
       def format_call(args)
@@ -199,21 +274,22 @@ module Clacky
           end
         end
 
-        # Handle text file reading
+        # Handle text file reading (including parser-extracted documents)
         lines = result[:lines_read] || result['lines_read'] || 0
         truncated = result[:truncated] || result['truncated']
-        "Read #{lines} lines#{truncated ? ' (truncated)' : ''}"
+        parsed_from = result[:parsed_from] || result['parsed_from']
+        suffix = parsed_from ? " (from #{parsed_from})" : ""
+        "Read #{lines} lines#{suffix}#{truncated ? ' (truncated)' : ''}"
       end
 
-      # Format result for LLM - handles both text and binary (image/PDF) content
+      # Format result for LLM - handles both text and binary (image) content
       # This method is called by the agent to format tool results before sending to LLM
       def format_result_for_llm(result)
-        # For LLM-compatible binary files with base64 data, return as content blocks
+        # For LLM-compatible binary files with base64 data (images only — documents
+        # are converted to text upstream via FileProcessor parsers).
         if result[:binary] && result[:base64_data]
-          # Create a text description
           description = "File: #{result[:path]}\nType: #{result[:format]}\nSize: #{format_file_size(result[:size_bytes])}"
 
-          # Add size warning for large files
           if result[:size_bytes] > Utils::FileProcessor::MAX_FILE_SIZE
             description += "\nWARNING: Large file (>#{Utils::FileProcessor::MAX_FILE_SIZE / 1024}KB) - may consume significant tokens"
           end
@@ -229,8 +305,6 @@ module Clacky
           #
           # The agent detects `:image_inject` in the tool result after observe() and
           # appends a `role: "user"` system_injected message containing the image block.
-          # This matches the standard workaround used by OpenAI's own agent SDK and
-          # pydantic-ai for multimodal tool outputs.
           if result[:mime_type]&.start_with?("image/")
             return {
               type: "text",
@@ -243,14 +317,14 @@ module Clacky
             }
           end
 
-          # For PDFs and other binary formats, just return metadata with base64
+          # No non-image binary type should reach here anymore — documents now
+          # go through the parser + text path. Keep this as a defensive fallback.
           return {
             type: "document",
             path: result[:path],
             format: result[:format],
             size_bytes: result[:size_bytes],
             mime_type: result[:mime_type],
-            base64_data: result[:base64_data],
             description: description
           }
         end
@@ -261,45 +335,35 @@ module Clacky
         # For directory listings, return as-is (no raw file content to preserve)
         return result if result[:is_directory]
 
-        # For plain text files: return a plain string so the agent sends it
-        # directly to the LLM without JSON-encoding (avoids \" / \n escaping).
+        # For plain text files (and parser-extracted documents): return a plain
+        # string so the agent sends it directly to the LLM without JSON-encoding
+        # (avoids \" / \n escaping).
         header = "File: #{result[:path]}"
+        if result[:parsed_from]
+          header += " [extracted from #{result[:parsed_from]}]"
+        end
         header += " (lines #{result[:start_line]}-#{result[:end_line]})" if result[:start_line]
         header += " [#{result[:lines_read]}/#{result[:total_lines]} lines]"
         header += " [TRUNCATED]" if result[:truncated]
         "#{header}\n\n#{result[:content]}"
       end
 
-      private def handle_binary_file(path)
-        # Check if it's a supported format using FileProcessor
-        if Utils::FileProcessor.supported_binary_file?(path)
-          # Use FileProcessor to convert to base64
-          begin
-            result = Utils::FileProcessor.file_to_base64(path)
-            {
-              path: path,
-              binary: true,
-              format: result[:format],
-              mime_type: result[:mime_type],
-              size_bytes: result[:size_bytes],
-              base64_data: result[:base64_data],
-              error: nil
-            }
-          rescue ArgumentError => e
-            # File too large or other error
-            file_size = File.size(path)
-            ext = File.extname(path).downcase
-            {
-              path: path,
-              binary: true,
-              format: ext.empty? ? "unknown" : ext[1..-1],
-              size_bytes: file_size,
-              content: nil,
-              error: e.message
-            }
-          end
-        else
-          # Binary file that we can't send to LLM
+      # Handle an image file: convert to base64 and return an LLM-ready result
+      # with the image_inject sidecar. Used by execute() for :image type files.
+      private def handle_image_file(path)
+        begin
+          result = Utils::FileProcessor.file_to_base64(path)
+          {
+            path: path,
+            binary: true,
+            format: result[:format],
+            mime_type: result[:mime_type],
+            size_bytes: result[:size_bytes],
+            base64_data: result[:base64_data],
+            error: nil
+          }
+        rescue ArgumentError => e
+          # File too large or unreadable
           file_size = File.size(path)
           ext = File.extname(path).downcase
           {
@@ -308,11 +372,52 @@ module Clacky
             format: ext.empty? ? "unknown" : ext[1..-1],
             size_bytes: file_size,
             content: nil,
-            error: "Binary file detected. This format cannot be read as text. File size: #{format_file_size(file_size)}"
+            error: e.message
           }
         end
       end
 
+      # Handle an unsupported binary file (no parser available, not an image).
+      # Returns a clear error message so the LLM knows it needs a different approach.
+      private def handle_unsupported_binary(path, ref = nil)
+        file_size = File.size(path)
+        ext = File.extname(path).downcase
+        {
+          path: path,
+          binary: true,
+          format: ext.empty? ? "unknown" : ext[1..-1],
+          size_bytes: file_size,
+          content: nil,
+          error: "Binary file detected. This format cannot be read as text. File size: #{format_file_size(file_size)}"
+        }
+      end
+
+      # Build an error result when the parser for a supported document format
+      # failed. The LLM receives the parser path so it can fix and retry, matching
+      # the behaviour of the file-upload pipeline (agent.rb's file_prompt).
+      private def build_parser_failure_result(path, ref)
+        ext = File.extname(path).downcase
+        file_size = File.size(path) rescue 0
+        message_lines = ["Failed to extract text from #{ext.empty? ? 'file' : ext[1..-1].upcase}."]
+        message_lines << "Parser error: #{ref.parse_error}" if ref.parse_error
+        if ref.parser_path
+          expected_preview = "#{path}.preview.md"
+          message_lines << "Parser script: #{ref.parser_path}"
+          message_lines << "To fix: edit the parser, then run: ruby #{ref.parser_path} #{path} > #{expected_preview}"
+          message_lines << "After a successful parse, re-run file_reader on this file."
+        end
+        {
+          path: path,
+          binary: true,
+          format: ext.empty? ? "unknown" : ext[1..-1],
+          size_bytes: file_size,
+          content: nil,
+          parser_path: ref.parser_path,
+          parse_error: ref.parse_error,
+          error: message_lines.join("\n")
+        }
+      end
+
       private def detect_mime_type(path, data)
         Utils::FileProcessor.detect_mime_type(path, data)
       end