smart_rag 0.1.0

data/db/fix_search_issues.sql

@@ -0,0 +1,81 @@
+-- Database fix script for SmartRAG search issues
+-- This script fixes three bugs:
+-- 1. Incorrect jieba configuration name
+-- 2. Wrong language in existing data
+-- 3. Rebuild fulltext indexes with correct tokenizer
+
+BEGIN;
+
+-- Fix 1: Update text_search_configs to use correct jieba config name
+UPDATE text_search_configs
+SET config_name = 'jiebacfg'
+WHERE config_name = 'jieba';
+
+-- Verify the fix
+SELECT 'Fixed text_search_configs: Changed jieba to jiebacfg' AS status,
+       COUNT(*) as updated_rows
+FROM text_search_configs
+WHERE config_name = 'jiebacfg';
+
+-- Fix 2: Detect and update language for source_documents
+-- For documents with Chinese content, set language to 'zh'
+-- First, identify documents with Chinese content based on sections
+WITH chinese_sections AS (
+  SELECT DISTINCT ss.document_id
+  FROM source_sections ss
+  WHERE ss.content ~ '[\u4e00-\u9fff]'
+)
+UPDATE source_documents sd
+SET language = 'zh'
+WHERE sd.id IN (SELECT document_id FROM chinese_sections)
+  AND (sd.language = 'en' OR sd.language IS NULL OR sd.language = '');
+
+-- Verify the fix
+SELECT 'Fixed source_documents language: Set to zh for Chinese documents' AS status,
+       COUNT(*) as updated_docs
+FROM source_documents
+WHERE language = 'zh';
+
+-- Fix 3: Rebuild fulltext indexes
+-- Delete existing fulltext indexes
+DELETE FROM section_fts;
+
+-- Rebuild fulltext indexes using the trigger
+-- The trigger will use the updated language and correct config name
+INSERT INTO section_fts (section_id, document_id, language, fts_title, fts_content, fts_combined)
+SELECT
+  ss.id,
+  ss.document_id,
+  COALESCE(sd.language, 'zh'),
+  NULL,
+  NULL,
+  NULL
+FROM source_sections ss
+JOIN source_documents sd ON sd.id = ss.document_id;
+
+-- Now update the fts fields by calling the trigger
+-- This will cause the trigger to fire and rebuild with correct language/tokenizer
+UPDATE source_sections SET updated_at = CURRENT_TIMESTAMP;
+
+COMMIT;
+
+-- Verification queries
+-- Check updated text_search_configs
+SELECT '=== Verification: text_search_configs ===' AS info;
+SELECT language_code, config_name FROM text_search_configs WHERE language_code LIKE 'zh%';
+
+-- Check updated source_documents
+SELECT '=== Verification: source_documents ===' AS info;
+SELECT id, title, language FROM source_documents;
+
+-- Check rebuilt section_fts
+SELECT '=== Verification: section_fts sample ===' AS info;
+SELECT section_id, language FROM section_fts LIMIT 5;
+
+-- Test fulltext search with Chinese
+SELECT '=== Test: Fulltext search with Chinese ===' AS info;
+SELECT ss.section_title, ssf.language
+FROM section_fts ssf
+JOIN source_sections ss ON ss.id = ssf.section_id
+WHERE ssf.fts_combined @@ to_tsquery('jiebacfg', '小动物')
+LIMIT 3;

data/db/migrations/001_create_source_documents.rb

@@ -0,0 +1,26 @@
+Sequel.migration do
+  up do
+    create_table :source_documents do
+      primary_key :id
+      String :title, null: false, size: 255
+      String :url, text: true
+      String :author, size: 255
+      Date :publication_date
+      String :language, size: 10, default: 'en'
+      String :description, text: true
+      # 0: pending, 1: completed, 2: failed
+      Integer :download_state, default: 0, null: false
+      DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
+      DateTime :updated_at, default: Sequel::CURRENT_TIMESTAMP
+    end
+
+    # Add index for common queries
+    add_index :source_documents, :download_state
+    add_index :source_documents, :created_at
+    add_index :source_documents, :language
+  end
+
+  down do
+    drop_table :source_documents
+  end
+end

data/db/migrations/002_create_source_sections.rb

@@ -0,0 +1,20 @@
+Sequel.migration do
+  up do
+    create_table :source_sections do
+      primary_key :id
+      foreign_key :document_id, :source_documents, null: false, on_delete: :cascade
+      String :content, text: true, null: false
+      String :section_title, size: 500
+      Integer :section_number, default: 0
+      DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
+      DateTime :updated_at, default: Sequel::CURRENT_TIMESTAMP
+    end
+
+    add_index :source_sections, :document_id
+    add_index :source_sections, :section_number
+  end
+
+  down do
+    drop_table :source_sections
+  end
+end

data/db/migrations/003_create_tags.rb

@@ -0,0 +1,17 @@
+Sequel.migration do
+  up do
+    create_table :tags do
+      primary_key :id
+      String :name, null: false, unique: true, size: 255
+      foreign_key :parent_id, :tags, on_delete: :set_null
+      DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
+    end
+
+    add_index :tags, :name
+    add_index :tags, :parent_id
+  end
+
+  down do
+    drop_table :tags
+  end
+end

data/db/migrations/004_create_research_topics.rb

@@ -0,0 +1,16 @@
+Sequel.migration do
+  up do
+    create_table :research_topics do
+      primary_key :id
+      String :name, null: false, unique: true, size: 255
+      String :description, text: true
+      DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
+    end
+
+    add_index :research_topics, :name
+  end
+
+  down do
+    drop_table :research_topics
+  end
+end

data/db/migrations/005_create_relationship_tables.rb

@@ -0,0 +1,42 @@
+Sequel.migration do
+  up do
+    # Section tags (many-to-many)
+    create_table :section_tags do
+      foreign_key :section_id, :source_sections, null: false, on_delete: :cascade
+      foreign_key :tag_id, :tags, null: false, on_delete: :cascade
+      primary_key [:section_id, :tag_id]
+      DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
+    end
+
+    add_index :section_tags, [:section_id, :tag_id], unique: true
+    add_index :section_tags, :tag_id
+
+    # Research topic sections (many-to-many)
+    create_table :research_topic_sections do
+      foreign_key :research_topic_id, :research_topics, null: false, on_delete: :cascade
+      foreign_key :section_id, :source_sections, null: false, on_delete: :cascade
+      primary_key [:research_topic_id, :section_id]
+      DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
+    end
+
+    add_index :research_topic_sections, [:research_topic_id, :section_id], unique: true
+    add_index :research_topic_sections, :section_id
+
+    # Research topic tags (many-to-many)
+    create_table :research_topic_tags do
+      foreign_key :research_topic_id, :research_topics, null: false, on_delete: :cascade
+      foreign_key :tag_id, :tags, null: false, on_delete: :cascade
+      primary_key [:research_topic_id, :tag_id]
+      DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
+    end
+
+    add_index :research_topic_tags, [:research_topic_id, :tag_id], unique: true
+    add_index :research_topic_tags, :tag_id
+  end
+
+  down do
+    drop_table :research_topic_tags
+    drop_table :research_topic_sections
+    drop_table :section_tags
+  end
+end

data/db/migrations/006_create_text_search_configs.rb

@@ -0,0 +1,28 @@
+Sequel.migration do
+  up do
+    create_table :text_search_configs do
+      String :language_code, primary_key: true
+      String :config_name, null: false
+      TrueClass :is_installed, default: true
+    end
+
+    # Add index for config name lookup
+    add_index :text_search_configs, :config_name
+
+    # Seed initial language configurations
+    # Note: These assume pg_jieba extension is installed for Chinese support
+    from(:text_search_configs).multi_insert(
+      [
+        { language_code: 'en', config_name: 'pg_catalog.english' },
+        { language_code: 'zh', config_name: 'jiebacfg' },
+        { language_code: 'ja', config_name: 'pg_catalog.simple' },
+        { language_code: 'ko', config_name: 'pg_catalog.simple' },
+        { language_code: 'default', config_name: 'pg_catalog.simple' }
+      ]
+    )
+  end
+
+  down do
+    drop_table :text_search_configs
+  end
+end

data/db/migrations/007_create_section_fts.rb

@@ -0,0 +1,109 @@
+Sequel.migration do
+  up do
+    create_table :section_fts do
+      # One-to-one relationship with source_sections
+      primary_key :section_id
+      foreign_key :document_id, :source_documents, on_delete: :cascade
+      String :language, null: false, default: 'en'
+      column :fts_title, 'tsvector'
+      column :fts_content, 'tsvector'
+      column :fts_combined, 'tsvector'
+      DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
+      DateTime :updated_at, default: Sequel::CURRENT_TIMESTAMP
+    end
+
+    # Primary composite index for combined search
+    add_index :section_fts, :fts_combined, type: :gin
+    add_index :section_fts, :language
+    add_index :section_fts, :fts_title, type: :gin
+
+    # Partitioned indexes for frequently used languages
+    # These improve query performance for language-specific searches
+    run 'CREATE INDEX section_fts_gin_zh ON section_fts USING GIN (fts_combined) WHERE language = \'zh\''
+    run 'CREATE INDEX section_fts_gin_en ON section_fts USING GIN (fts_combined) WHERE language = \'en\''
+    run 'CREATE INDEX section_fts_gin_ja ON section_fts USING GIN (fts_combined) WHERE language = \'ja\''
+
+    # Foreign key index
+    add_index :section_fts, :document_id
+
+    # Create trigger function to automatically maintain FTS data
+    run <<-SQL
+      CREATE OR REPLACE FUNCTION update_section_fts()
+      RETURNS TRIGGER AS $$
+      DECLARE
+          v_language TEXT;
+          v_config TEXT;
+      BEGIN
+          -- Get document language
+          SELECT COALESCE(sd.language, 'en') INTO v_language
+          FROM source_documents sd
+          WHERE sd.id = NEW.document_id;
+
+          -- Get corresponding text search configuration
+          SELECT COALESCE(tsc.config_name, 'pg_catalog.simple') INTO v_config
+          FROM text_search_configs tsc
+          WHERE tsc.language_code = v_language;
+
+          -- Maintain full-text search data
+          INSERT INTO section_fts (section_id, document_id, language, fts_title, fts_content, fts_combined)
+          VALUES (
+              NEW.id,
+              NEW.document_id,
+              v_language,
+              setweight(to_tsvector(v_config::regconfig, COALESCE(NEW.section_title,'')), 'A'),
+              setweight(to_tsvector(v_config::regconfig, COALESCE(NEW.content,'')), 'B'),
+              setweight(to_tsvector(v_config::regconfig, COALESCE(NEW.section_title,'')), 'A') ||
+              setweight(to_tsvector(v_config::regconfig, COALESCE(NEW.content,'')), 'B')
+          )
+          ON CONFLICT (section_id) DO UPDATE SET
+              document_id = NEW.document_id,
+              language = v_language,
+              fts_title = setweight(to_tsvector(v_config::regconfig, COALESCE(NEW.section_title,'')), 'A'),
+              fts_content = setweight(to_tsvector(v_config::regconfig, COALESCE(NEW.content,'')), 'B'),
+              fts_combined = setweight(to_tsvector(v_config::regconfig, COALESCE(NEW.section_title,'')), 'A') ||
+                            setweight(to_tsvector(v_config::regconfig, COALESCE(NEW.content,'')), 'B'),
+              updated_at = CURRENT_TIMESTAMP;
+
+          RETURN NEW;
+      END;
+      $$ LANGUAGE plpgsql;
+    SQL
+
+    # Create trigger on source_sections table
+    run <<-SQL
+      CREATE TRIGGER trigger_update_section_fts
+          AFTER INSERT OR UPDATE ON source_sections
+          FOR EACH ROW EXECUTE FUNCTION update_section_fts();
+    SQL
+
+    # Also create trigger for document language changes
+    run <<-SQL
+      CREATE OR REPLACE FUNCTION update_section_fts_on_doc_update()
+      RETURNS TRIGGER AS $$
+      BEGIN
+          -- Update FTS for all sections when document language changes
+          IF NEW.language IS DISTINCT FROM OLD.language THEN
+              UPDATE section_fts
+              SET updated_at = CURRENT_TIMESTAMP
+              WHERE document_id = NEW.id;
+          END IF;
+          RETURN NEW;
+      END;
+      $$ LANGUAGE plpgsql;
+    SQL
+
+    run <<-SQL
+      CREATE TRIGGER trigger_update_section_fts_on_doc
+          AFTER UPDATE OF language ON source_documents
+          FOR EACH ROW EXECUTE FUNCTION update_section_fts_on_doc_update();
+    SQL
+  end
+
+  down do
+    run 'DROP TRIGGER IF EXISTS trigger_update_section_fts_on_doc ON source_documents'
+    run 'DROP FUNCTION IF EXISTS update_section_fts_on_doc_update()'
+    run 'DROP TRIGGER IF EXISTS trigger_update_section_fts ON source_sections'
+    run 'DROP FUNCTION IF EXISTS update_section_fts()'
+    drop_table :section_fts
+  end
+end

data/db/migrations/008_create_embeddings.rb

@@ -0,0 +1,28 @@
+Sequel.migration do
+  up do
+    # Create vector extension if it doesn't exist
+    run 'CREATE EXTENSION IF NOT EXISTS vector'
+
+    create_table :embeddings do
+      primary_key :id
+      foreign_key :source_id, :source_sections, null: false, on_delete: :cascade
+      # Vector dimension size (adjust based on embedding model)
+      column :vector, 'vector(1024)', null: false
+      DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
+    end
+
+    # Create IVFFLAT index for approximate nearest neighbor search
+    # IVFFLAT provides good accuracy with fast search
+    run 'CREATE INDEX idx_embedding_vector ON embeddings USING ivfflat (vector vector_cosine_ops) WITH (lists = 100)'
+
+    # Composite index for source_id lookups
+    add_index :embeddings, :source_id
+
+    # Additional index for faster lookups during similarity search
+    add_index :embeddings, :created_at
+  end
+
+  down do
+    drop_table :embeddings
+  end
+end

data/db/migrations/009_create_search_logs.rb

@@ -0,0 +1,30 @@
+Sequel.migration do
+  up do
+    create_table :search_logs do
+      primary_key :id
+      String :query, null: false, text: true
+      String :search_type, size: 20 # 'vector', 'fulltext', 'hybrid'
+      Integer :execution_time_ms
+      Integer :results_count
+      column :query_vector, 'vector(1024)' # Store query vector for analysis
+      column :result_ids, 'integer[]' # Store result IDs for relevance analysis
+      column :filters, 'jsonb' # Store search filters
+      DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
+    end
+
+    # Indexes for performance monitoring
+    add_index :search_logs, :created_at
+    add_index :search_logs, :search_type
+    add_index :search_logs, :execution_time_ms
+
+    # GIN index for full-text search on queries
+    run "CREATE INDEX search_logs_query_idx ON search_logs USING gin (to_tsvector('simple', query))"
+
+    # Composite index for common analytics queries
+    add_index :search_logs, [:search_type, :created_at]
+  end
+
+  down do
+    drop_table :search_logs
+  end
+end

data/db/migrations/010_add_metadata_to_source_documents.rb

@@ -0,0 +1,10 @@
+Sequel.migration do
+  up do
+    add_column :source_documents, :metadata, :jsonb, default: '{}'
+    add_index :source_documents, :metadata, type: :gin
+  end
+
+  down do
+    drop_column :source_documents, :metadata
+  end
+end

data/db/migrations/011_add_source_fields_to_source_documents.rb

@@ -0,0 +1,23 @@
+Sequel.migration do
+  up do
+    add_column :source_documents, :source_type, String, size: 50, default: 'manual'
+    add_column :source_documents, :source_uri, String, text: true
+    add_column :source_documents, :content_hash, String, size: 128
+
+    add_index :source_documents, :source_type
+    add_index :source_documents, :source_uri
+    add_index :source_documents, :content_hash
+    add_index :source_documents, [:source_uri, :content_hash]
+  end
+
+  down do
+    drop_index :source_documents, [:source_uri, :content_hash]
+    drop_index :source_documents, :content_hash
+    drop_index :source_documents, :source_uri
+    drop_index :source_documents, :source_type
+
+    drop_column :source_documents, :content_hash
+    drop_column :source_documents, :source_uri
+    drop_column :source_documents, :source_type
+  end
+end

data/db/rebuild_fts_complete.sql

@@ -0,0 +1,51 @@
+-- Complete rebuild of section_fts fulltext indexes
+-- This script rebuilds all fulltext search indexes with correct language and tokenizer
+
+BEGIN;
+
+-- 1. Delete all existing fts data
+DELETE FROM section_fts;
+
+-- 2. Insert all sections with basic info
+INSERT INTO section_fts (section_id, document_id, language)
+SELECT 
+  ss.id,
+  ss.document_id,
+  COALESCE(sd.language, 'zh') as language
+FROM source_sections ss
+JOIN source_documents sd ON sd.id = ss.document_id;
+
+-- 3. Trigger updates by touching all sections
+-- This will cause the trigger to fire and rebuild fts vectors
+-- We do this in batches to avoid locking
+UPDATE source_sections SET updated_at = CURRENT_TIMESTAMP WHERE id IN (
+  SELECT id FROM source_sections LIMIT 1000
+);
+
+UPDATE source_sections SET updated_at = CURRENT_TIMESTAMP WHERE id IN (
+  SELECT id FROM source_sections LIMIT 1000 OFFSET 1000
+);
+
+UPDATE source_sections SET updated_at = CURRENT_TIMESTAMP WHERE id IN (
+  SELECT id FROM source_sections LIMIT 1000 OFFSET 2000
+);
+
+-- Verify rebuild
+SELECT '=== Verification ===' as info;
+SELECT COUNT(*) as total_sections FROM section_fts;
+SELECT COUNT(*) as sections_with_fts_title FROM section_fts WHERE fts_title IS NOT NULL;
+SELECT COUNT(*) as sections_with_fts_content FROM section_fts WHERE fts_content IS NOT NULL;
+SELECT COUNT(*) as sections_with_fts_combined FROM section_fts WHERE fts_combined IS NOT NULL;
+
+-- Test search
+SELECT '=== Test Search for 老人情感 ===' as info;
+SELECT section_id, section_title
+FROM section_fts ssf
+JOIN source_sections ss ON ss.id = ssf.section_id
+WHERE ssf.fts_combined @@ plainto_tsquery('jiebacfg', '老人情感')
+  AND ssf.language = 'zh'
+LIMIT 5;
+
+COMMIT;
+
+SELECT '=== Complete! All fulltext indexes rebuilt ===' as status;

data/db/seeds/text_search_configs.sql

@@ -0,0 +1,28 @@
+-- Seed data for text_search_configs table
+-- This file contains the standard language configurations for full-text search
+
+-- Clear existing data
+TRUNCATE TABLE text_search_configs;
+
+-- Insert language configurations
+INSERT INTO text_search_configs (language_code, config_name, is_installed) VALUES
+('en', 'pg_catalog.english', true),
+('en_us', 'pg_catalog.english', true),
+('en_gb', 'pg_catalog.english', true),
+('zh', 'jiebacfg', true),
+('zh_cn', 'jiebacfg', true),
+('zh_tw', 'jiebacfg', true),
+('ja', 'pg_catalog.simple', true),
+('ja_jp', 'pg_catalog.simple', true),
+('ko', 'pg_catalog.simple', true),
+('ko_kr', 'pg_catalog.simple', true),
+('es', 'pg_catalog.spanish', true),
+('fr', 'pg_catalog.french', true),
+('de', 'pg_catalog.german', true),
+('it', 'pg_catalog.italian', true),
+('ru', 'pg_catalog.russian', true),
+('ar', 'pg_catalog.simple', true),
+('default', 'pg_catalog.simple', true);
+
+-- Add comment for jieba configuration
+COMMENT ON COLUMN text_search_configs.config_name IS 'Full-text search configuration name. For Chinese support, ensure pg_jieba extension is installed';

data/examples/01_quick_start.rb

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "common"
+
+include Examples::Common
+
+print_header("Quick Start")
+smart_rag = build_client
+
+stats = smart_rag.statistics
+puts "SmartRAG initialized."
+puts "Documents in DB: #{stats[:document_count]}"
+
+query = ARGV[0] || "machine learning algorithms"
+results = smart_rag.search(
+  query,
+  search_type: "hybrid",
+  limit: 5,
+  include_content: true,
+)
+
+print_header("First Search: #{query}")
+results.fetch(:results, []).each_with_index do |result, idx|
+  score = result[:combined_score] || result[:similarity] || 0.0
+  puts "#{idx + 1}. #{result[:section_title]} (score: #{score.round(3)})"
+  next unless result[:content]
+
+  preview = result[:content][0, 120].to_s.gsub(/\s+/, " ")
+  puts "   #{preview}..."
+end
+

data/examples/02_document_management.rb

@@ -0,0 +1,41 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "common"
+
+include Examples::Common
+
+print_header("Document Management")
+smart_rag = build_client
+
+document_path = ARGV[0]
+if document_path.nil? || document_path.strip.empty?
+  warn "Usage: ruby examples/02_document_management.rb /path/to/document.md"
+  exit 1
+end
+
+add_result = smart_rag.add_document(
+  document_path,
+  title: File.basename(document_path),
+  generate_embeddings: true,
+  generate_tags: true,
+  tags: ["example", "usage_examples"],
+  metadata: { source: "examples/02_document_management.rb" },
+)
+
+print_json("Add Result", add_result)
+
+document_id = add_result[:document_id]
+detail = smart_rag.get_document(document_id)
+print_json("Document Detail", detail || {})
+
+list = smart_rag.list_documents(page: 1, per_page: 10, search: File.basename(document_path))
+print_json("Document List", list)
+
+# Pass DELETE=1 to demonstrate cleanup:
+# DELETE=1 ruby examples/02_document_management.rb test/python_basics.md
+if ENV["DELETE"] == "1"
+  delete_result = smart_rag.remove_document(document_id)
+  print_json("Delete Result", delete_result)
+end
+

data/examples/03_search_operations.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "common"
+
+include Examples::Common
+
+print_header("Search Operations")
+smart_rag = build_client
+
+query = ARGV[0] || "deep learning applications in healthcare"
+
+hybrid = smart_rag.search(
+  query,
+  search_type: "hybrid",
+  limit: 5,
+  alpha: 0.7,
+  include_metadata: true,
+)
+print_json("Hybrid Search Metadata", hybrid[:metadata] || {})
+
+vector = smart_rag.vector_search(
+  "neural network architectures",
+  limit: 5,
+  include_content: false,
+)
+print_json("Vector Search Metadata", vector[:metadata] || {})
+
+fulltext = smart_rag.fulltext_search(
+  '"deep reinforcement learning"',
+  limit: 5,
+)
+print_json("Fulltext Search Metadata", fulltext[:metadata] || {})
+
+multilingual_queries = [
+  ["zh_cn", "人工智能应用"],
+  ["ja", "機械学習アルゴリズム"],
+  ["ko", "딥러닝 모델"],
+  ["auto", "AI和机器学习的发展"],
+]
+
+multilingual_queries.each do |language, q|
+  result = smart_rag.search(q, search_type: "hybrid", language: language, limit: 3)
+  puts "[#{language}] #{q} -> #{result.fetch(:results, []).length} results"
+end
+

data/examples/04_topics_and_tags.rb

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "common"
+
+include Examples::Common
+
+print_header("Topics And Tags")
+smart_rag = build_client
+
+topic = smart_rag.create_topic(
+  "AI in Healthcare",
+  "Applications of AI in medical diagnosis and treatment",
+  tags: ["ai", "healthcare", "diagnosis"],
+  document_ids: [],
+)
+print_json("Created Topic", topic)
+
+topic_detail = smart_rag.get_topic(topic[:id])
+print_json("Topic Detail", topic_detail || {})
+
+all_topics = smart_rag.list_topics(page: 1, per_page: 20)
+puts "Topic count on page: #{all_topics.fetch(:topics, []).length}"
+
+recommendations = smart_rag.get_topic_recommendations(topic[:id], limit: 5)
+print_json("Topic Recommendations", recommendations)
+
+sample_text = <<~TEXT
+  Machine learning is a subset of artificial intelligence that enables systems
+  to learn and improve from experience without explicit programming.
+TEXT
+
+tags = smart_rag.generate_tags(sample_text, context: "AI introduction", max_tags: 5)
+print_json("Generated Tags", tags)
+
+tag_page = smart_rag.list_tags(page: 1, per_page: 20)
+puts "Tag count on page: #{tag_page.fetch(:tags, []).length}"
+