agent-tome 1.0.0

data/lib/agent/tome/commands/create.rb

@@ -0,0 +1,168 @@
+require "active_support/core_ext/string/inflections"
+
+module Agent
+  module Tome
+    module Commands
+      class Create
+        TRACKING_PARAMS = %w[fbclid gclid fbid mc_cid mc_eid].freeze
+
+        def call(input)
+          validate!(input)
+
+          result = {}
+
+          ActiveRecord::Base.transaction do
+            article = Article.create!(
+              description: input["description"],
+              created_at: Time.now
+            )
+
+            entry = Entry.create!(
+              article: article,
+              body: input["body"],
+              created_at: Time.now
+            )
+
+            process_keywords!(article, input["keywords"] || [])
+            web_source_ids = process_web_sources!(entry, input["web_sources"] || [])
+            file_source_ids = process_file_sources!(entry, input["file_sources"] || [])
+            process_related_articles!(article, input["related_article_ids"] || [])
+
+            result = {
+              "article_global_id" => article.global_id,
+              "entry_global_id" => entry.global_id,
+              "web_source_global_ids" => web_source_ids,
+              "file_source_global_ids" => file_source_ids
+            }
+          end
+
+          result
+        end
+
+        private
+
+        def validate!(input)
+          raise ValidationError, "Missing description" unless input.key?("description")
+          raise ValidationError, "Missing body" unless input.key?("body")
+
+          desc = input["description"]
+          raise ValidationError, "description must be a string" unless desc.is_a?(String)
+          raise ValidationError, "description cannot be blank" if desc.strip.empty?
+          raise ValidationError, "description must be 350 characters or fewer" if desc.length > 350
+
+          body = input["body"]
+          raise ValidationError, "body must be a string" unless body.is_a?(String)
+          raise ValidationError, "body cannot be blank" if body.strip.empty?
+
+          validate_keywords!(input["keywords"]) if input.key?("keywords")
+          validate_web_sources!(input["web_sources"]) if input.key?("web_sources")
+          validate_file_sources!(input["file_sources"]) if input.key?("file_sources")
+          validate_related_ids!(input["related_article_ids"]) if input.key?("related_article_ids")
+        end
+
+        def validate_keywords!(keywords)
+          return unless keywords
+
+          raise ValidationError, "keywords must be an array" unless keywords.is_a?(Array)
+
+          keywords.each do |kw|
+            raise ValidationError, "keyword must be a non-empty string" unless kw.is_a?(String) && !kw.strip.empty?
+          end
+        end
+
+        def validate_web_sources!(sources)
+          return unless sources
+
+          raise ValidationError, "web_sources must be an array" unless sources.is_a?(Array)
+
+          sources.each do |src|
+            raise ValidationError, "web_source url is required" unless src.is_a?(Hash) && src["url"]
+            raise ValidationError, "invalid URL: #{src["url"]}" unless UrlNormalizer.valid?(src["url"])
+          end
+        end
+
+        def validate_file_sources!(sources)
+          return unless sources
+
+          raise ValidationError, "file_sources must be an array" unless sources.is_a?(Array)
+
+          sources.each do |src|
+            raise ValidationError, "file_source path cannot be empty" if src["path"].to_s.strip.empty?
+            raise ValidationError, "file_source system_name cannot be empty" if src["system_name"].to_s.strip.empty?
+          end
+        end
+
+        def validate_related_ids!(ids)
+          return unless ids
+
+          raise ValidationError, "related_article_ids must be an array" unless ids.is_a?(Array)
+
+          ids.each do |id|
+            raise ValidationError, "Referenced article not found: #{id}" unless Article.exists?(global_id: id)
+          end
+        end
+
+        def process_keywords!(article, keywords)
+          keywords.each do |kw|
+            normalized = normalize_keyword(kw)
+            keyword = Keyword.find_or_create_by!(term: normalized) do |k|
+              k.created_at = Time.now
+            end
+            ArticleKeyword.find_or_create_by!(article: article, keyword: keyword) do |ak|
+              ak.created_at = Time.now
+            end
+          end
+        end
+
+        def process_web_sources!(entry, sources)
+          sources.map do |src|
+            normalized_url = UrlNormalizer.normalize(src["url"])
+            ws = WebSource.find_or_create_by!(url: normalized_url) do |w|
+              w.global_id = GlobalId.generate
+              w.title = src["title"]
+              w.fetched_at = src["fetched_at"] ? Time.parse(src["fetched_at"]) : nil
+              w.created_at = Time.now
+            end
+            EntryWebSource.find_or_create_by!(entry: entry, web_source: ws) do |ews|
+              ews.created_at = Time.now
+            end
+            ws.global_id
+          end
+        end
+
+        def process_file_sources!(entry, sources)
+          sources.map do |src|
+            fs = FileSource.find_or_create_by!(path: src["path"], system_name: src["system_name"]) do |f|
+              f.global_id = GlobalId.generate
+              f.created_at = Time.now
+            end
+            EntryFileSource.find_or_create_by!(entry: entry, file_source: fs) do |efs|
+              efs.created_at = Time.now
+            end
+            fs.global_id
+          end
+        end
+
+        def process_related_articles!(article, related_ids)
+          related_ids.each do |target_id|
+            raise ValidationError, "An article cannot reference itself" if target_id == article.global_id
+
+            target = Article.find_by!(global_id: target_id)
+            ArticleReference.find_or_create_by!(
+              source_article: article,
+              target_article: target
+            ) do |ref|
+              ref.created_at = Time.now
+            end
+          end
+        end
+
+        def normalize_keyword(kw)
+          words = kw.downcase.split("-")
+          words[-1] = ActiveSupport::Inflector.singularize(words[-1])
+          words.join("-")
+        end
+      end
+    end
+  end
+end

data/lib/agent/tome/commands/fetch.rb

@@ -0,0 +1,69 @@
+module Agent
+  module Tome
+    module Commands
+      class Fetch
+        def initialize(global_id:)
+          @global_id = global_id
+        end
+
+        def call
+          article = Article.find_by(global_id: @global_id)
+          raise NotFoundError, "Article not found: #{@global_id}" unless article
+
+          result = {
+            "global_id" => article.global_id,
+            "description" => article.description,
+            "keywords" => article.keywords.pluck(:term).sort,
+            "created_at" => article.created_at.iso8601,
+            "entries" => format_entries(article)
+          }
+
+          if (link = article.consolidation_as_new)
+            old = link.old_article
+            result["consolidated_from"] = {
+              "global_id" => old.global_id,
+              "description" => old.description
+            }
+          end
+
+          result
+        end
+
+        private
+
+        def format_entries(article)
+          article.entries.order(:created_at).map do |entry|
+            {
+              "global_id" => entry.global_id,
+              "body" => entry.body,
+              "created_at" => entry.created_at.iso8601,
+              "web_sources" => format_web_sources(entry),
+              "file_sources" => format_file_sources(entry)
+            }
+          end
+        end
+
+        def format_web_sources(entry)
+          entry.web_sources.map do |ws|
+            {
+              "global_id" => ws.global_id,
+              "url" => ws.url,
+              "title" => ws.title,
+              "fetched_at" => ws.fetched_at&.iso8601
+            }
+          end
+        end
+
+        def format_file_sources(entry)
+          entry.file_sources.map do |fs|
+            {
+              "global_id" => fs.global_id,
+              "path" => fs.path,
+              "system_name" => fs.system_name
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/agent/tome/commands/keywords_list.rb

@@ -0,0 +1,22 @@
+module Agent
+  module Tome
+    module Commands
+      class KeywordsList
+        def initialize(prefix:)
+          @prefix = prefix
+        end
+
+        def call
+          raise ValidationError, "A prefix/substring argument is required" if @prefix.nil? || @prefix.strip.empty?
+
+          terms = Keyword
+            .where("LOWER(term) LIKE ?", "%#{@prefix.downcase}%")
+            .order(:term)
+            .pluck(:term)
+
+          { "keywords" => terms }
+        end
+      end
+    end
+  end
+end

data/lib/agent/tome/commands/related.rb

@@ -0,0 +1,80 @@
+module Agent
+  module Tome
+    module Commands
+      class Related
+        def initialize(global_id:)
+          @global_id = global_id
+        end
+
+        def call
+          article = Article.find_by(global_id: @global_id)
+          raise NotFoundError, "Article not found: #{@global_id}" unless article
+
+          {
+            "shared_keywords" => find_shared_keywords(article),
+            "references_to" => find_references_to(article),
+            "referenced_by" => find_referenced_by(article),
+            "consolidated_from" => find_consolidated_from(article),
+            "consolidated_into" => find_consolidated_into(article)
+          }
+        end
+
+        private
+
+        def find_shared_keywords(article)
+          keyword_ids = article.keywords.pluck(:id)
+          return [] if keyword_ids.empty?
+
+          Article
+            .joins(:article_keywords)
+            .where(article_keywords: { keyword_id: keyword_ids })
+            .where.not(id: article.id)
+            .group("articles.id")
+            .select("articles.*, COUNT(DISTINCT article_keywords.keyword_id) AS shared_keyword_count")
+            .order("shared_keyword_count DESC")
+            .limit(100)
+            .map { |a| format_article(a, shared_keyword_count: a.shared_keyword_count.to_i) }
+        end
+
+        def find_references_to(article)
+          ArticleReference
+            .where(source_article: article)
+            .includes(:target_article)
+            .map { |ref| format_article(ref.target_article) }
+        end
+
+        def find_referenced_by(article)
+          ArticleReference
+            .where(target_article: article)
+            .includes(:source_article)
+            .map { |ref| format_article(ref.source_article) }
+        end
+
+        def find_consolidated_from(article)
+          ConsolidationLink
+            .where(new_article: article)
+            .includes(:old_article)
+            .map { |link| format_article(link.old_article) }
+        end
+
+        def find_consolidated_into(article)
+          ConsolidationLink
+            .where(old_article: article)
+            .includes(:new_article)
+            .map { |link| format_article(link.new_article) }
+        end
+
+        def format_article(article, extra = {})
+          base = {
+            "global_id" => article.global_id,
+            "description" => article.description,
+            "keywords" => article.keywords.pluck(:term).sort,
+            "created_at" => article.created_at.iso8601
+          }
+          base.merge!(extra.transform_keys(&:to_s)) if extra.any?
+          base
+        end
+      end
+    end
+  end
+end

data/lib/agent/tome/commands/search.rb

@@ -0,0 +1,62 @@
+require "active_support/core_ext/string/inflections"
+
+module Agent
+  module Tome
+    module Commands
+      class Search
+        def initialize(keywords:, match: "any")
+          @keywords = keywords
+          @match = match
+        end
+
+        def call
+          raise ValidationError, "At least one keyword is required" if @keywords.empty?
+
+          normalized = @keywords.map { |kw| normalize_keyword(kw) }
+          keyword_ids = Keyword.where(term: normalized).pluck(:id)
+
+          return { "results" => [] } if keyword_ids.empty?
+
+          articles = find_matching_articles(keyword_ids, normalized)
+
+          {
+            "results" => articles.first(1000).map { |row| format_result(row) }
+          }
+        end
+
+        private
+
+        def find_matching_articles(keyword_ids, normalized_terms)
+          base = Article
+            .joins(:article_keywords)
+            .where(article_keywords: { keyword_id: keyword_ids })
+            .group("articles.id")
+            .select("articles.*, COUNT(DISTINCT article_keywords.keyword_id) AS matching_keyword_count")
+            .order("matching_keyword_count DESC")
+
+          if @match == "all"
+            base = base.having("COUNT(DISTINCT article_keywords.keyword_id) = ?", keyword_ids.length)
+          end
+
+          base.limit(1000)
+        end
+
+        def format_result(article)
+          {
+            "global_id" => article.global_id,
+            "description" => article.description,
+            "keywords" => article.keywords.pluck(:term).sort,
+            "matching_keyword_count" => article.matching_keyword_count.to_i,
+            "created_at" => article.created_at.iso8601
+          }
+        end
+
+        def normalize_keyword(kw)
+          words = kw.downcase.split("-")
+          words[-1] = ActiveSupport::Inflector.singularize(words[-1])
+          words.join("-")
+        end
+      end
+    end
+  end
+end

data/lib/agent/tome/commands/source_search.rb

@@ -0,0 +1,60 @@
+module Agent
+  module Tome
+    module Commands
+      class SourceSearch
+        def initialize(source:, system: nil)
+          @source = source
+          @system = system
+        end
+
+        def call
+          articles = if url?(@source)
+            search_by_url
+          else
+            search_by_path
+          end
+
+          {
+            "results" => articles.map { |a| format_article(a) }
+          }
+        end
+
+        private
+
+        def url?(str)
+          str.start_with?("http://", "https://")
+        end
+
+        def search_by_url
+          normalized = UrlNormalizer.normalize(@source)
+          ws = WebSource.find_by(url: normalized)
+          return [] unless ws
+
+          Article
+            .joins(entries: :web_sources)
+            .where(web_sources: { id: ws.id })
+            .distinct
+        end
+
+        def search_by_path
+          scope = Article
+            .joins(entries: :file_sources)
+            .where(file_sources: { path: @source })
+
+          scope = scope.where(file_sources: { system_name: @system }) if @system
+
+          scope.distinct
+        end
+
+        def format_article(article)
+          {
+            "global_id" => article.global_id,
+            "description" => article.description,
+            "keywords" => article.keywords.pluck(:term).sort,
+            "created_at" => article.created_at.iso8601
+          }
+        end
+      end
+    end
+  end
+end

data/lib/agent/tome/config.rb

@@ -0,0 +1,52 @@
+require "yaml"
+require "fileutils"
+
+module Agent
+  module Tome
+    class Config
+      DEFAULT_CONFIG_DIR = File.expand_path("~/.agent-tome")
+
+      attr_reader :db_path, :config_dir
+
+      def initialize(config_dir: nil)
+        @config_dir = config_dir || ENV.fetch("AGENT_TOME_CONFIG_DIR", DEFAULT_CONFIG_DIR)
+      end
+
+      def load!
+        if File.directory?(@config_dir)
+          read_config!
+        else
+          bootstrap!
+          read_config!
+        end
+        self
+      end
+
+      private
+
+      def bootstrap!
+        FileUtils.mkdir_p(@config_dir)
+        default_db = File.join(@config_dir, "tome.db")
+        File.write(config_file_path, YAML.dump("db_path" => default_db))
+      end
+
+      def config_file_path
+        File.join(@config_dir, "config.yml")
+      end
+
+      def read_config!
+        raise ConfigError, "Config file not found: #{config_file_path}" unless File.exist?(config_file_path)
+
+        data = YAML.load_file(config_file_path)
+
+        unless data.is_a?(Hash) && data.key?("db_path") && !data["db_path"].to_s.strip.empty?
+          raise ConfigError, "db_path is not configured in #{config_file_path}"
+        end
+
+        @db_path = data["db_path"]
+      end
+    end
+
+    class ConfigError < StandardError; end
+  end
+end

data/lib/agent/tome/database.rb

@@ -0,0 +1,77 @@
+require "active_record"
+require "fileutils"
+
+module Agent
+  module Tome
+    module Database
+      MIGRATIONS_PATH = File.expand_path("../../../../db/migrate", __FILE__)
+
+      @migrations_path_override = nil
+
+      class << self
+        def migrations_path
+          @migrations_path_override || MIGRATIONS_PATH
+        end
+
+        def migrations_path=(path)
+          @migrations_path_override = path
+        end
+      end
+
+      def self.connect!(db_path)
+        db_dir = File.dirname(db_path)
+
+        unless File.directory?(db_dir)
+          begin
+            FileUtils.mkdir_p(db_dir)
+          rescue Errno::EACCES, Errno::EPERM, Errno::EROFS => e
+            raise DatabaseError, "Database path is not writable: #{db_path} (#{e.message})"
+          end
+        end
+
+        unless writable_path?(db_path)
+          raise DatabaseError, "Database path is not writable: #{db_path}"
+        end
+
+        ActiveRecord::Base.logger = nil
+
+        ActiveRecord::Base.establish_connection(
+          adapter: "sqlite3",
+          database: db_path
+        )
+
+        ActiveRecord::Base.connection.execute("PRAGMA foreign_keys = ON")
+        ActiveRecord::Base.connection.execute("PRAGMA journal_mode = WAL")
+        ActiveRecord::Base.connection.execute("PRAGMA busy_timeout = 5000")
+
+        run_migrations!
+      rescue Errno::EACCES, Errno::EPERM, Errno::EROFS => e
+        raise DatabaseError, "Database path is not writable: #{db_path} (#{e.message})"
+      end
+
+      def self.disconnect!
+        ActiveRecord::Base.remove_connection
+      rescue StandardError
+        nil
+      end
+
+      def self.run_migrations!
+        ActiveRecord::Migration.verbose = false
+        context = ActiveRecord::MigrationContext.new(migrations_path)
+        context.migrate
+      end
+
+      def self.writable_path?(path)
+        if File.exist?(path)
+          File.writable?(path)
+        else
+          dir = File.dirname(path)
+          File.directory?(dir) && File.writable?(dir)
+        end
+      end
+      private_class_method :writable_path?
+    end
+
+    class DatabaseError < StandardError; end
+  end
+end

data/lib/agent/tome/global_id.rb

@@ -0,0 +1,16 @@
+module Agent
+  module Tome
+    module GlobalId
+      BASE58_ALPHABET = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
+      PATTERN = /\A[1-9A-HJ-NP-Za-km-z]{7}\z/
+
+      def self.generate
+        Array.new(7) { BASE58_ALPHABET[rand(58)] }.join
+      end
+
+      def self.valid?(id)
+        id.is_a?(String) && PATTERN.match?(id)
+      end
+    end
+  end
+end

data/lib/agent/tome/models/application_record.rb

@@ -0,0 +1,8 @@
+module Agent
+  module Tome
+    class ApplicationRecord < ActiveRecord::Base
+      self.abstract_class = true
+      self.table_name_prefix = ""
+    end
+  end
+end

data/lib/agent/tome/models/article.rb

@@ -0,0 +1,36 @@
+module Agent
+  module Tome
+    class Article < ApplicationRecord
+      has_many :entries, dependent: :destroy
+      has_many :article_keywords, dependent: :destroy
+      has_many :keywords, through: :article_keywords
+      has_many :source_references,
+               class_name: "Agent::Tome::ArticleReference",
+               foreign_key: :source_article_id,
+               dependent: :destroy
+      has_many :target_references,
+               class_name: "Agent::Tome::ArticleReference",
+               foreign_key: :target_article_id,
+               dependent: :destroy
+      has_one :consolidation_as_new,
+              class_name: "Agent::Tome::ConsolidationLink",
+              foreign_key: :new_article_id
+      has_one :consolidation_as_old,
+              class_name: "Agent::Tome::ConsolidationLink",
+              foreign_key: :old_article_id
+
+      validates :global_id, presence: true, length: { is: 7 }, uniqueness: true
+      validates :description, presence: true, length: { maximum: 350 }
+
+      before_validation :assign_global_id, on: :create
+
+      private
+
+      def assign_global_id
+        return if global_id.present?
+
+        self.global_id = GlobalId.generate
+      end
+    end
+  end
+end

data/lib/agent/tome/models/article_keyword.rb

@@ -0,0 +1,10 @@
+module Agent
+  module Tome
+    class ArticleKeyword < ApplicationRecord
+      belongs_to :article
+      belongs_to :keyword
+
+      validates :article_id, uniqueness: { scope: :keyword_id }
+    end
+  end
+end

data/lib/agent/tome/models/article_reference.rb

@@ -0,0 +1,20 @@
+module Agent
+  module Tome
+    class ArticleReference < ApplicationRecord
+      belongs_to :source_article, class_name: "Agent::Tome::Article"
+      belongs_to :target_article, class_name: "Agent::Tome::Article"
+
+      validates :source_article_id, uniqueness: { scope: :target_article_id }
+      validate :not_self_referencing
+
+      private
+
+      def not_self_referencing
+        return unless source_article_id && target_article_id
+        return unless source_article_id == target_article_id
+
+        errors.add(:base, "An article cannot reference itself")
+      end
+    end
+  end
+end