aircana 1.0.0 → 1.2.0

data/lib/aircana/cli/commands/agents.rb

@@ -3,6 +3,8 @@
 require "json"
 require "tty-prompt"
 require_relative "../../generators/agents_generator"
+require_relative "../../contexts/manifest"
+require_relative "../../contexts/web"
 
 module Aircana
   module CLI
@@ -13,7 +15,7 @@ module Aircana
       class << self # rubocop:disable Metrics/ClassLength
         def refresh(agent)
           normalized_agent = normalize_string(agent)
-          perform_refresh(normalized_agent)
+          perform_manifest_aware_refresh(normalized_agent)
         rescue Aircana::Error => e
           handle_refresh_error(normalized_agent, e)
         end
@@ -42,6 +44,9 @@ module Aircana
           # Prompt for knowledge fetching
           prompt_for_knowledge_fetch(prompt, normalized_agent_name)
 
+          # Prompt for web URL fetching
+          prompt_for_url_fetch(prompt, normalized_agent_name)
+
           # Prompt for agent file review
           prompt_for_agent_review(prompt, file)
 
@@ -58,13 +63,52 @@ module Aircana
           print_agents_list(agent_folders)
         end
 
+        def add_url(agent, url) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/PerceivedComplexity
+          normalized_agent = normalize_string(agent)
+
+          unless agent_exists?(normalized_agent)
+            Aircana.human_logger.error "Agent '#{agent}' not found. Use 'aircana agents list' to see available agents."
+            exit 1
+          end
+
+          web = Aircana::Contexts::Web.new
+          result = web.fetch_url_for(agent: normalized_agent, url: url)
+
+          if result
+            # Update manifest with the new URL
+            existing_sources = Aircana::Contexts::Manifest.sources_from_manifest(normalized_agent)
+            web_sources = existing_sources.select { |s| s["type"] == "web" }
+            other_sources = existing_sources.reject { |s| s["type"] == "web" }
+
+            if web_sources.any?
+              # Add to existing web source
+              web_sources.first["urls"] << result
+            else
+              # Create new web source
+              web_sources = [{ "type" => "web", "urls" => [result] }]
+            end
+
+            all_sources = other_sources + web_sources
+            Aircana::Contexts::Manifest.update_manifest(normalized_agent, all_sources)
+
+            Aircana.human_logger.success "Successfully added URL to agent '#{agent}'"
+          else
+            Aircana.human_logger.error "Failed to fetch URL: #{url}"
+            exit 1
+          end
+        rescue Aircana::Error => e
+          Aircana.human_logger.error "Failed to add URL: #{e.message}"
+          exit 1
+        end
+
         private
 
         def perform_refresh(normalized_agent)
           confluence = Aircana::Contexts::Confluence.new
-          pages_count = confluence.fetch_pages_for(agent: normalized_agent)
+          result = confluence.fetch_pages_for(agent: normalized_agent)
 
-          log_refresh_result(normalized_agent, pages_count)
+          log_refresh_result(normalized_agent, result[:pages_count])
+          result
         end
 
         def log_refresh_result(normalized_agent, pages_count)
@@ -75,6 +119,49 @@ module Aircana
           end
         end
 
+        def perform_manifest_aware_refresh(normalized_agent) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+          total_pages = 0
+          all_sources = []
+
+          # Try manifest-based refresh first
+          if Aircana::Contexts::Manifest.manifest_exists?(normalized_agent)
+            Aircana.human_logger.info "Refreshing from knowledge manifest..."
+
+            # Refresh Confluence sources
+            confluence = Aircana::Contexts::Confluence.new
+            confluence_result = confluence.refresh_from_manifest(agent: normalized_agent)
+            total_pages += confluence_result[:pages_count]
+            all_sources.concat(confluence_result[:sources])
+
+            # Refresh web sources
+            web = Aircana::Contexts::Web.new
+            web_result = web.refresh_web_sources(agent: normalized_agent)
+            total_pages += web_result[:pages_count]
+            all_sources.concat(web_result[:sources])
+          else
+            Aircana.human_logger.info "No manifest found, falling back to label-based search..."
+            confluence = Aircana::Contexts::Confluence.new
+            confluence_result = confluence.fetch_pages_for(agent: normalized_agent)
+            total_pages += confluence_result[:pages_count]
+            all_sources.concat(confluence_result[:sources])
+          end
+
+          # Update manifest with all sources combined
+          Aircana::Contexts::Manifest.update_manifest(normalized_agent, all_sources) if all_sources.any?
+
+          log_refresh_result(normalized_agent, total_pages)
+          { pages_count: total_pages, sources: all_sources }
+        end
+
+        def show_gitignore_recommendation
+          Aircana.human_logger.info ""
+          Aircana.human_logger.info "💡 Recommendation: Add knowledge directories to .gitignore:"
+          Aircana.human_logger.info "   echo \".aircana/agents/*/knowledge/\" >> .gitignore"
+          Aircana.human_logger.info ""
+          Aircana.human_logger.info "   This keeps knowledge sources in version control while excluding"
+          Aircana.human_logger.info "   the actual knowledge content from your repository."
+        end
+
         def log_no_pages_found(normalized_agent)
           Aircana.human_logger.info "No pages found for agent '#{normalized_agent}'. " \
                                     "Make sure pages are labeled with '#{normalized_agent}' in Confluence."
@@ -114,7 +201,8 @@ module Aircana
 
           if prompt.yes?("Would you like to fetch knowledge for this agent from Confluence now?")
             Aircana.human_logger.info "Fetching knowledge from Confluence..."
-            perform_refresh(normalized_agent_name)
+            result = perform_refresh(normalized_agent_name)
+            show_gitignore_recommendation if result[:pages_count]&.positive?
           else
             Aircana.human_logger.info(
               "Skipping knowledge fetch. You can run 'aircana agents refresh #{normalized_agent_name}' later."
@@ -125,6 +213,43 @@ module Aircana
           Aircana.human_logger.info "You can try again later with 'aircana agents refresh #{normalized_agent_name}'"
         end
 
+        def prompt_for_url_fetch(prompt, normalized_agent_name) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+          return unless prompt.yes?("Would you like to add web URLs for this agent's knowledge base?")
+
+          urls = []
+          loop do
+            url = prompt.ask("Enter URL (or press Enter to finish):")
+            break if url.nil? || url.strip.empty?
+
+            url = url.strip
+            if valid_url?(url)
+              urls << url
+            else
+              Aircana.human_logger.warn "Invalid URL format: #{url}. Please enter a valid HTTP or HTTPS URL."
+            end
+          end
+
+          return if urls.empty?
+
+          begin
+            Aircana.human_logger.info "Fetching #{urls.size} URL(s)..."
+            web = Aircana::Contexts::Web.new
+            result = web.fetch_urls_for(agent: normalized_agent_name, urls: urls)
+
+            if result[:pages_count].positive?
+              Aircana.human_logger.success "Successfully fetched #{result[:pages_count]} URL(s)"
+              show_gitignore_recommendation
+            else
+              Aircana.human_logger.warn "No URLs were successfully fetched"
+            end
+          rescue Aircana::Error => e
+            Aircana.human_logger.warn "Failed to fetch URLs: #{e.message}"
+            Aircana.human_logger.info(
+              "You can add URLs later with 'aircana agents add-url #{normalized_agent_name} <URL>'"
+            )
+          end
+        end
+
         def prompt_for_agent_review(prompt, file_path)
           Aircana.human_logger.info "Agent file created at: #{file_path}"
 
@@ -186,6 +311,18 @@ module Aircana
           config["description"] || "No description available"
         end
 
+        def agent_exists?(agent_name)
+          agent_dir = File.join(Aircana.configuration.agent_knowledge_dir, agent_name)
+          Dir.exist?(agent_dir)
+        end
+
+        def valid_url?(url)
+          uri = URI.parse(url)
+          %w[http https].include?(uri.scheme) && !uri.host.nil?
+        rescue URI::InvalidURIError
+          false
+        end
+
         def find_available_editor
           %w[code subl atom nano vim vi].find { |cmd| system("which #{cmd} > /dev/null 2>&1") }
         end

data/lib/aircana/contexts/confluence.rb

@@ -3,6 +3,7 @@
 require "httparty"
 require "reverse_markdown"
 require_relative "local"
+require_relative "manifest"
 require_relative "confluence_logging"
 require_relative "confluence_http"
 require_relative "confluence_content"
@@ -28,10 +29,35 @@ module Aircana
         setup_httparty
 
         pages = search_and_log_pages(agent)
-        return 0 if pages.empty?
+        return { pages_count: 0, sources: [] } if pages.empty?
 
-        process_pages(pages, agent)
-        pages.size
+        sources = process_pages_with_manifest(pages, agent)
+        create_or_update_manifest(agent, sources)
+
+        { pages_count: pages.size, sources: sources }
+      end
+
+      def refresh_from_manifest(agent:)
+        sources = Manifest.sources_from_manifest(agent)
+        return { pages_count: 0, sources: [] } if sources.empty?
+
+        validate_configuration!
+        setup_httparty
+
+        confluence_sources = sources.select { |s| s["type"] == "confluence" }
+        return { pages_count: 0, sources: [] } if confluence_sources.empty?
+
+        all_pages = []
+        confluence_sources.each do |source|
+          pages = fetch_pages_from_source(source)
+          all_pages.concat(pages)
+        end
+
+        return { pages_count: 0, sources: [] } if all_pages.empty?
+
+        updated_sources = process_pages_with_manifest(all_pages, agent)
+
+        { pages_count: all_pages.size, sources: updated_sources }
       end
 
       def search_and_log_pages(agent)
@@ -48,8 +74,54 @@ module Aircana
         end
       end
 
+      def process_pages_with_manifest(pages, agent)
+        page_metadata = []
+
+        ProgressTracker.with_batch_progress(pages, "Processing pages") do |page, _index|
+          store_page_as_markdown(page, agent)
+          page_metadata << extract_page_metadata(page)
+        end
+
+        build_source_metadata(agent, page_metadata)
+      end
+
       private
 
+      def fetch_pages_from_source(source)
+        case source["type"]
+        when "confluence"
+          fetch_pages_by_label(source["label"])
+        else
+          []
+        end
+      end
+
+      def extract_page_metadata(page)
+        {
+          "id" => page["id"],
+          "title" => page["title"],
+          "last_updated" => page.dig("version", "when") || Time.now.utc.strftime("%Y-%m-%dT%H:%M:%SZ")
+        }
+      end
+
+      def build_source_metadata(agent, page_metadata)
+        [
+          {
+            "type" => "confluence",
+            "label" => agent,
+            "pages" => page_metadata
+          }
+        ]
+      end
+
+      def create_or_update_manifest(agent, sources)
+        if Manifest.manifest_exists?(agent)
+          Manifest.update_manifest(agent, sources)
+        else
+          Manifest.create_manifest(agent, sources)
+        end
+      end
+
       def validate_configuration!
         config = Aircana.configuration
 

data/lib/aircana/contexts/manifest.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+
+module Aircana
+  module Contexts
+    class Manifest
+      class << self
+        def create_manifest(agent, sources)
+          validate_sources(sources)
+
+          manifest_path = manifest_path_for(agent)
+          manifest_data = build_manifest_data(agent, sources)
+
+          FileUtils.mkdir_p(File.dirname(manifest_path))
+          File.write(manifest_path, JSON.pretty_generate(manifest_data))
+
+          Aircana.human_logger.info "Created knowledge manifest for agent '#{agent}'"
+          manifest_path
+        end
+
+        def update_manifest(agent, sources)
+          validate_sources(sources)
+
+          manifest_path = manifest_path_for(agent)
+
+          if File.exist?(manifest_path)
+            existing_data = JSON.parse(File.read(manifest_path))
+            manifest_data = existing_data.merge({
+                                                  "last_updated" => Time.now.utc.strftime("%Y-%m-%dT%H:%M:%SZ"),
+                                                  "sources" => sources
+                                                })
+          else
+            manifest_data = build_manifest_data(agent, sources)
+          end
+
+          FileUtils.mkdir_p(File.dirname(manifest_path))
+          File.write(manifest_path, JSON.pretty_generate(manifest_data))
+          manifest_path
+        end
+
+        def read_manifest(agent)
+          manifest_path = manifest_path_for(agent)
+          return nil unless File.exist?(manifest_path)
+
+          begin
+            manifest_data = JSON.parse(File.read(manifest_path))
+            validate_manifest(manifest_data)
+            manifest_data
+          rescue JSON::ParserError => e
+            Aircana.human_logger.warn "Invalid manifest for agent '#{agent}': #{e.message}"
+            nil
+          rescue ManifestError => e
+            Aircana.human_logger.warn "Manifest validation failed for agent '#{agent}': #{e.message}"
+            nil
+          end
+        end
+
+        def sources_from_manifest(agent)
+          manifest = read_manifest(agent)
+          return [] unless manifest
+
+          manifest["sources"] || []
+        end
+
+        def manifest_exists?(agent)
+          File.exist?(manifest_path_for(agent))
+        end
+
+        private
+
+        def manifest_path_for(agent)
+          resolved_agent_path = resolve_agent_path(agent)
+          File.join(resolved_agent_path, "manifest.json")
+        end
+
+        def resolve_agent_path(agent)
+          base_path = File.join(Aircana.configuration.agent_knowledge_dir, agent)
+
+          # If this is a symlink (multi-root scenario), resolve to original
+          if File.symlink?(base_path)
+            File.readlink(base_path)
+          else
+            base_path
+          end
+        end
+
+        def build_manifest_data(agent, sources)
+          timestamp = Time.now.utc.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+          {
+            "version" => "1.0",
+            "agent" => agent,
+            "created" => timestamp,
+            "last_updated" => timestamp,
+            "sources" => sources
+          }
+        end
+
+        def validate_manifest(manifest_data)
+          required_fields = %w[version agent sources]
+
+          required_fields.each do |field|
+            raise ManifestError, "Missing required field: #{field}" unless manifest_data.key?(field)
+          end
+
+          unless manifest_data["version"] == "1.0"
+            raise ManifestError, "Unsupported manifest version: #{manifest_data["version"]}"
+          end
+
+          validate_sources(manifest_data["sources"])
+        end
+
+        def validate_sources(sources)
+          raise ManifestError, "Sources must be an array" unless sources.is_a?(Array)
+
+          sources.each do |source|
+            validate_source(source)
+          end
+        end
+
+        def validate_source(source)
+          raise ManifestError, "Each source must be a hash" unless source.is_a?(Hash)
+
+          raise ManifestError, "Source missing required field: type" unless source.key?("type")
+
+          case source["type"]
+          when "confluence"
+            validate_confluence_source(source)
+          when "web"
+            validate_web_source(source)
+          else
+            raise ManifestError, "Unknown source type: #{source["type"]}"
+          end
+        end
+
+        def validate_confluence_source(source)
+          raise ManifestError, "Confluence source missing required field: label" unless source.key?("label")
+
+          return unless source.key?("pages") && !source["pages"].is_a?(Array)
+
+          raise ManifestError, "Confluence pages must be an array"
+        end
+
+        def validate_web_source(source)
+          raise ManifestError, "Web source missing required field: urls" unless source.key?("urls")
+
+          raise ManifestError, "Web urls must be an array" unless source["urls"].is_a?(Array)
+
+          source["urls"].each do |url_entry|
+            validate_web_url_entry(url_entry)
+          end
+        end
+
+        def validate_web_url_entry(url_entry)
+          raise ManifestError, "Each URL entry must be a hash" unless url_entry.is_a?(Hash)
+
+          raise ManifestError, "URL entry missing required field: url" unless url_entry.key?("url")
+
+          raise ManifestError, "URL entry missing required field: title" unless url_entry.key?("title")
+        end
+      end
+    end
+
+    class ManifestError < StandardError; end
+  end
+end