arcadedb 0.5.2 → 0.5.4

data/examples/rag_usecase_demo.rb

@@ -0,0 +1,345 @@
+# RAG Use Case Demo - Knowledge Base Search
+#
+# This example demonstrates a complete RAG (Retrieval-Augmented Generation) workflow
+# using a knowledge base of technical documentation.
+#
+# Features demonstrated:
+# - Document ingestion with vector embeddings
+# - Vector similarity search for semantic queries
+# - Hybrid search combining vector + full-text
+# - Category and tag-based filtering
+# - Real-world query scenarios
+#
+# Run with: ruby examples/rag_usecase_demo.rb
+
+require 'bundler/setup'
+require 'arcade'
+
+# =============================================================================
+# Model Definition
+# =============================================================================
+
+module UseCase
+  class KnowledgeDocument < Arcade::Vertex
+    include Arcade::Ragschema
+
+    attribute :title, Types::String
+    attribute :content, Types::String
+    attribute :embedding?, Types::Array.of(Types::Float).optional
+    attribute :category, Types::String.optional
+    attribute :tags, Types::Array.of(Types::String).optional
+    attribute :author, Types::String.optional
+    attribute :source_url, Types::String.optional
+    attribute :created_at, Types::DateTime.optional
+    attribute :updated_at, Types::DateTime.optional
+
+    def self.db_init
+      rag_db_init(
+        vector_dimension: 1536,
+        full_text_fields: ['title', 'content', 'category']
+      )
+    end
+  end
+
+  # =============================================================================
+  # Helper for generating mock embeddings (for demo purposes)
+  # In production, use a real embedding service (OpenAI, HuggingFace, etc.)
+  # =============================================================================
+
+  module EmbeddingHelper
+    ##
+    # Generate a deterministic mock embedding for demonstration
+    #
+    # @param text [String] Text to embed
+    # @param seed [Integer] Seed for deterministic generation
+    # @return [Array<Float>] 1536-dimensional vector
+    def self.generate_mock_embedding(text, seed: 42)
+      base_hash = (text + seed.to_s).hash
+      vector = []
+      1536.times do |i|
+        value = ((base_hash * (i + 1) * 31) % 10000) / 10000.0 - 0.5
+        vector << value
+      end
+      vector
+    end
+  end
+end
+
+# =============================================================================
+# Demo Data - Technical Knowledge Base
+# =============================================================================
+
+KNOWLEDGE_BASE = [
+  {
+    title: 'Introduction to Graph Databases',
+    content: 'Graph databases are designed to store and treat relationships between data entities as first-class citizens. Unlike relational databases, graph databases use nodes and edges to represent and store data. This makes them ideal for analyzing complex relationships and patterns in data.',
+    category: 'Database Fundamentals',
+    tags: ['graph', 'database', 'nosql', 'fundamentals'],
+    author: 'Database Team'
+  },
+  {
+    title: 'ArcadeDB Overview',
+    content: 'ArcadeDB is a multi-model graph database built from the ground up with support for vertices, edges, documents, arrays, and vectors. It provides a SQL-like query language and supports RAG (Retrieval-Augmented Generation) workloads with native vector search capabilities.',
+    category: 'Database Systems',
+    tags: ['arcadedb', 'graph', 'vector', 'rag'],
+    author: 'ArcadeDB Team'
+  },
+  {
+    title: 'Vector Search Fundamentals',
+    content: 'Vector search, also known as semantic search, uses mathematical representations of data called embeddings. These embeddings capture the semantic meaning of text, allowing for similarity-based searches that understand context rather than just keyword matching.',
+    category: 'Search Technology',
+    tags: ['vector', 'embedding', 'similarity', 'ai'],
+    author: 'AI Research Team'
+  },
+  {
+    title: 'RAG Architecture Patterns',
+    content: 'Retrieval-Augmented Generation (RAG) combines retrieval-based and generation-based approaches. It first retrieves relevant documents from a knowledge base using vector similarity, then uses a language model to generate responses based on the retrieved context.',
+    category: 'AI Architecture',
+    tags: ['rag', 'llm', 'retrieval', 'generation'],
+    author: 'ML Engineering'
+  },
+  {
+    title: 'SQL Query Optimization',
+    content: 'Query optimization in SQL databases involves techniques like indexing, query planning, and execution strategy selection. Proper indexing strategies can dramatically improve query performance, especially for complex joins and aggregations.',
+    category: 'Database Performance',
+    tags: ['sql', 'optimization', 'indexing', 'performance'],
+    author: 'Performance Team'
+  },
+  {
+    title: 'NoSQL vs SQL Databases',
+    content: 'NoSQL databases offer flexible schemas and horizontal scaling, making them suitable for unstructured data and high-velocity applications. SQL databases provide ACID transactions and structured querying capabilities. The choice depends on your specific use case requirements.',
+    category: 'Database Comparison',
+    tags: ['nosql', 'sql', 'comparison', 'architecture'],
+    author: 'Architecture Team'
+  },
+  {
+    title: 'Full-Text Search Implementation',
+    content: 'Full-text search indexes allow for efficient text querying beyond simple LIKE matches. They support tokenization, stemming, and relevance scoring. Modern search engines combine full-text search with vector embeddings for hybrid search capabilities.',
+    category: 'Search Technology',
+    tags: ['fulltext', 'search', 'indexing', 'hybrid'],
+    author: 'Search Team'
+  },
+  {
+    title: 'Embedding Models for Text',
+    content: 'Text embedding models like BERT, Sentence Transformers, and OpenAI embeddings convert text into dense vectors. These vectors capture semantic relationships, enabling tasks like semantic search, clustering, and similarity detection.',
+    category: 'Machine Learning',
+    tags: ['embedding', 'bert', 'nlp', 'transformers'],
+    author: 'NLP Team'
+  }
+].freeze
+
+# =============================================================================
+# Demo Functions
+# =============================================================================
+
+def print_header(title)
+  puts "\n" + '=' * 70
+  puts title.center(70)
+  puts '=' * 70
+end
+
+def print_section(title)
+  puts "\n--- #{title} ---\n"
+end
+
+def print_document(doc, show_similarity: false)
+  puts "  Title: #{doc.title}"
+  puts "  Category: #{doc.category}"
+  puts "  Author: #{doc.author}"
+  puts "  Tags: #{doc.tags.join(', ')}"
+  puts "  Content: #{doc.content[0..150]}..."
+  puts "  Similarity: #{doc.similarity.round(4)}" if doc.respond_to?(:similarity) && doc.similarity
+  puts
+end
+
+def load_knowledge_base
+  print_section('Loading Knowledge Base')
+
+  KNOWLEDGE_BASE.each do |doc_data|
+    embedding = UseCase::EmbeddingHelper.generate_mock_embedding(
+      doc_data[:title] + ' ' + doc_data[:content]
+    )
+
+    UseCase::KnowledgeDocument.create(
+      title: doc_data[:title],
+      content: doc_data[:content],
+      embedding: embedding,
+      category: doc_data[:category],
+      tags: doc_data[:tags],
+      author: doc_data[:author],
+      created_at: DateTime.now
+    )
+
+    puts "  ✓ Loaded: #{doc_data[:title]}"
+  end
+
+  total = UseCase::KnowledgeDocument.count
+  puts "\nTotal documents: #{total}"
+end
+
+def demo_vector_search
+  print_section('Vector Similarity Search')
+
+  queries = [
+    'What is a graph database and how does it work?',
+    'How does RAG architecture work?',
+    'Explain vector embeddings for text search'
+  ]
+
+  queries.each do |query|
+    puts "\nQuery: \"#{query}\""
+    puts "-" * 50
+
+    query_embedding = UseCase::EmbeddingHelper.generate_mock_embedding(query)
+
+    results = UseCase::KnowledgeDocument.vector_search(
+      query_embedding,
+      limit: 3,
+      threshold: 0.0
+    )
+
+    results.each_with_index do |doc, i|
+      puts "  [#{i + 1}] #{doc.title} (similarity: #{doc.similarity.round(4) if doc.similarity})"
+    end
+  end
+end
+
+def demo_hybrid_search
+  print_section('Hybrid Search (Vector + Full-Text)')
+
+  searches = [
+    { query: 'database performance', text: 'optimization' },
+    { query: 'search technology', text: 'Full-Text' },
+    { query: 'machine learning', text: 'embedding' }
+  ]
+
+  searches.each do |search|
+    puts "\nVector Query: \"#{search[:query]}\""
+    puts "Text Filter: \"#{search[:text]}\""
+    puts "-" * 50
+
+    query_embedding = UseCase::EmbeddingHelper.generate_mock_embedding(search[:query])
+
+    results = UseCase::KnowledgeDocument.hybrid_search(
+      query_embedding,
+      search[:text],
+      limit: 5,
+      vector_threshold: 0.0
+    )
+
+    if results.empty?
+      puts "  No results found"
+    else
+      results.each_with_index do |doc, i|
+        puts "  [#{i + 1}] #{doc.title}"
+      end
+    end
+  end
+end
+
+def demo_category_filtering
+  print_section('Category-Based Filtering')
+
+  categories = ['Search Technology', 'Database Systems', 'Machine Learning']
+
+  categories.each do |category|
+    puts "\nCategory: #{category}"
+    puts "-" * 50
+
+    results = UseCase::KnowledgeDocument.where("category = '#{category}'")
+
+    results.each do |doc|
+      puts "  • #{doc.title} (#{doc.author})"
+    end
+  end
+end
+
+def demo_tag_search
+  print_section('Tag-Based Search')
+
+  tags = ['vector', 'graph', 'embedding']
+
+  tags.each do |tag|
+    puts "\nTag: #{tag}"
+    puts "-" * 50
+
+    results = UseCase::KnowledgeDocument.where("tags CONTAINS '#{tag}'")
+
+    results.each do |doc|
+      puts "  • #{doc.title} [#{doc.tags.join(', ')}]"
+    end
+  end
+end
+
+def demo_real_world_queries
+  print_section('Real-World Query Scenarios')
+
+  scenarios = {
+    'What is ArcadeDB?' => 'Definition query',
+    'difference between SQL and NoSQL' => 'Comparison query',
+    'How to optimize database queries?' => 'How-to query',
+    'Explain RAG benefits' => 'Concept exploration'
+  }
+
+  scenarios.each do |query, type|
+    puts "\n[#{type}] \"#{query}\""
+    puts "-" * 50
+
+    query_embedding = UseCase::EmbeddingHelper.generate_mock_embedding(query)
+
+    results = UseCase::KnowledgeDocument.vector_search(
+      query_embedding,
+      limit: 2,
+      threshold: 0.0
+    )
+
+    results.each do |doc|
+      puts "  → #{doc.title}"
+    end
+  end
+end
+
+# =============================================================================
+# Main Demo Runner
+# =============================================================================
+
+def run_demo
+  print_header('RAG Use Case Demo - Knowledge Base Search')
+
+  # Connect to database
+  Arcade::Init.connect('test')
+
+  # Create type if needed
+  unless Arcade::Init.db.types.include?('UseCase_KnowledgeDocument')
+    puts 'Creating KnowledgeDocument type...'
+    UseCase::KnowledgeDocument.create_type
+  end
+
+  # Clear existing data for fresh demo
+  UseCase::KnowledgeDocument.delete_all rescue nil
+
+  # Run demos
+  load_knowledge_base
+  demo_vector_search
+  demo_hybrid_search
+  demo_category_filtering
+  demo_tag_search
+  demo_real_world_queries
+
+  print_header('Demo Complete!')
+  puts "\nThis demo showcased:"
+  puts "  ✓ Document ingestion with vector embeddings"
+  puts "  ✓ Vector similarity search for semantic queries"
+  puts "  ✓ Hybrid search (vector + full-text)"
+  puts "  ✓ Category and tag-based filtering"
+  puts "  ✓ Real-world query scenarios"
+  puts "\nFor more information, see:"
+  puts "  - RAG_SCHEMA.md"
+  puts "  - spec/lib/usecase/rag_integration_spec.rb"
+  puts
+end
+
+# Run if executed directly
+if $0 == __FILE__
+  run_demo
+end

data/lib/arcade/api/primitives.rb

@@ -79,6 +79,49 @@ module Arcade
   #    response.raise_for_status
     end
 
+    # ------------------------------  post text ----------------------------------------------------- #
+    # posts a plain-text body (used by the TimeSeries Line Protocol endpoint)
+    def post_text command, body
+      case response = http.post( Config.base_uri + command, body: body )
+      in {status: 200..299}
+        response.status
+       in {status: 400..}
+        answer = response.json( symbolize_names: true )
+        answer[:detail] ||= answer[:error]   # ts-endpoints report error only
+        raise Arcade::QueryError.new **answer
+      else
+        raise response
+      end
+    end
+
+    # ------------------------------  post json raw ------------------------------------------------- #
+    # posts a json body, returns the parsed body (not the :result part)
+    def post_json command, payload
+      case response = http.post( Config.base_uri + command, json: payload )
+      in {status: 200..299}
+        JSON.parse( response.body, symbolize_names: true )
+       in {status: 400..}
+        answer = response.json( symbolize_names: true )
+        answer[:detail] ||= answer[:error]
+        raise Arcade::QueryError.new **answer
+      else
+        raise response
+      end
+    end
+
+    # ------------------------------  get json ------------------------------------------------------ #
+    # GET returning the parsed body (not only the :result part)
+    def get_json command
+      case response = http.get( Config.base_uri + command )
+      in {status: 200..299}
+        JSON.parse( response.body, symbolize_names: true )
+       in {status: 400..}
+        raise Arcade::QueryError.new **response.json( symbolize_names: true )
+      else
+        raise response
+      end
+    end
+
     # ------------------------------ transaction      ------------------------------------------------- #
     #
     # The payload, optional as a JSON, accepts the following parameters:

data/lib/arcade/base.rb

@@ -53,7 +53,7 @@ module Arcade
         begin
         loop do
           if the_class.respond_to?(:demodulize)
-            if [ 'Document','Vertex', 'Edge'].include?(the_class.demodulize)
+            if [ 'Document','Vertex', 'Edge', 'Timeseries'].include?(the_class.demodulize)
               if  the_class == superclass  # no inheritance
                   db.create_type the_class.demodulize, to_s.snake_case
               else

data/lib/arcade/database.rb

@@ -186,7 +186,7 @@ module Arcade
 
       content_params = params.except( :type, :bucket, :index, :from, :return, :session_id )
       target_params = params.slice( :type, :bucket, :index )
-#      session_id = params[:session_id]   # extraxt session_id  --> future-use? 
+#      session_id = params[:session_id]   # extraxt session_id  --> future-use?
       if  target_params.empty?
         raise "Could not insert: target missing (type:, bucket:, index:)"
       elsif content_params.empty?
@@ -195,7 +195,15 @@ module Arcade
         content = "CONTENT #{ content_params.to_json }"
         target =  target_params.map{|y,z|  y==:type ?  z : "#{y.to_s} #{ z } "}.join
         result =  Api.execute( database, session_id: session ){ "INSERT INTO #{target} #{content} "}
-        result &.first.allocate_model(false)
+        # 
+        result &.record&.allocate_model(false)
+        ## commented for now. reactivate if necessary (performance issue)
+        #record = result&.first
+        # Ensure @type is present for proper model allocation
+        #if record && !record.key?(:@type) && target_params[:type]
+        #  record = record.merge( :"@type" => target_params[:type] )
+        #end
+        #record&.allocate_model(false)
       end
     end
 

data/lib/arcade/rag_schema.rb

@@ -0,0 +1,131 @@
+module Arcade
+  ##
+  # Ragschema - A generic class to implement a RAG (Retrieval-Augmented Generation)
+  # compatible database schema for ArcadeDB.
+  #
+  # This module provides a default-index configuration optimized for:
+  # - Vector similarity search (semantic embeddings)
+  # - Metadata filtering
+  #
+  # Usage:
+  #   class MyDocument < Arcade::Vertex
+  #     include Arcade::Ragschema
+  #     attribute :content, Types::String
+  #     attribute :metadata, Types::Hash.optional
+  #   end
+  #
+  #   MyDocument.create_type
+  #
+  module Ragschema
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      ##
+      # Returns the default RAG schema setup commands
+      # to be executed after type creation
+      #
+      # @param vector_dimension [Integer] Dimension of vector embeddings (default: 1536 for OpenAI)
+      # @return [String] SQL commands for index creation
+      def rag_schema(vector_dimension: 1536)
+        type_name = database_name
+        commands = []
+
+        # Vector index for semantic similarity search
+        commands << "CREATE INDEX ON #{type_name} (embedding) LSM_VECTOR METADATA {dimensions: #{vector_dimension}, similarity: 'COSINE'}"
+
+        # Metadata index for filtering
+        commands << "CREATE INDEX IF NOT EXISTS ON #{type_name} (metadata) NOTUNIQUE"
+
+        commands.join("\n")
+      end
+
+      ##
+      # Default db_init method for RAG-compatible models
+      # Override this in your model class to customize
+      #
+      # @return [String] SQL commands to be executed on database
+      def rag_db_init(vector_dimension: 1536)
+        <<~SQL
+          CREATE PROPERTY #{database_name}.embedding ARRAY_OF_FLOATS
+          CREATE PROPERTY #{database_name}.content STRING
+          CREATE PROPERTY #{database_name}.metadata MAP
+          CREATE PROPERTY #{database_name}.source_url STRING
+          CREATE PROPERTY #{database_name}.created_at DATETIME
+          #{rag_schema(vector_dimension: vector_dimension)}
+        SQL
+      end
+
+      ##
+      # Perform vector similarity search across all documents
+      # Returns metadata only (not embedding vectors) for efficiency
+      #
+      # @param query_embedding [Array<Float>] The embedding vector to search for
+      # @param limit [Integer] Maximum number of results (default: 10)
+      # @param threshold [Float] Minimum similarity score (default: 0.7)
+      # @return [Array<Hash>] Matching records with metadata and similarity score
+      def vector_search(query_embedding, limit: 10, threshold: 0.7)
+        raise "vector_search must be called on a model class" unless respond_to?(:database_name)
+
+        type_name = database_name
+        # Convert array to ArcadeDB array literal format: [1.0,2.0,3.0]
+        embedding_array = "[#{query_embedding.join(',')}]"
+
+        # Select metadata fields only (not embedding) for efficiency
+        # Include @rid for reference and all other fields except embedding
+        query = <<~SQL
+          SELECT @rid, title, content, category, tags, author, source_url, created_at, updated_at, distance AS similarity
+          FROM (
+            SELECT expand(vectorNeighbors('#{type_name}[embedding]', #{embedding_array}, #{limit}))
+          )
+          WHERE distance <= (1.0 - #{threshold})
+          ORDER BY distance ASC
+        SQL
+
+        db.query(query)
+      end
+
+      ##
+      # Perform hybrid search combining vector similarity with SQL LIKE filtering
+      # Returns metadata only (not embedding vectors) for efficiency
+      #
+      # @param query_embedding [Array<Float>] The embedding vector
+      # @param text_query [String] SQL LIKE search query
+      # @param limit [Integer] Maximum number of results (default: 10)
+      # @param vector_threshold [Float] Minimum vector similarity (default: 0.7)
+      # @return [Array<Hash>] Matching records with metadata and similarity score
+      def hybrid_search(query_embedding, text_query, limit: 10, vector_threshold: 0.7)
+        raise "hybrid_search must be called on a model class" unless respond_to?(:database_name)
+
+        type_name = database_name
+        # Convert array to ArcadeDB array literal format: [1.0,2.0,3.0]
+        embedding_array = "[#{query_embedding.join(',')}]"
+
+        # Select metadata fields only (not embedding) for efficiency
+        query = <<~SQL
+          SELECT @rid, title, content, category, tags, author, source_url, created_at, updated_at, distance AS similarity
+          FROM (
+            SELECT expand(vectorNeighbors('#{type_name}[embedding]', #{embedding_array}, #{limit}))
+          )
+          WHERE distance <= (1.0 - #{vector_threshold})
+            AND content LIKE '%#{text_query}%'
+          ORDER BY distance ASC
+        SQL
+
+        db.query(query)
+      end
+    end
+
+    ##
+    # Instance method to perform vector similarity search
+    #
+    # @param query_embedding [Array<Float>] The embedding vector to search for
+    # @param limit [Integer] Maximum number of results (default: 10)
+    # @param threshold [Float] Minimum similarity score (default: 0.7)
+    # @return [Array<Base>] Matching records
+    def vector_search(query_embedding, limit: 10, threshold: 0.7)
+      self.class.vector_search(query_embedding, limit: limit, threshold: threshold)
+    end
+  end
+end

data/lib/arcade/support/model.rb

@@ -60,7 +60,18 @@ module Arcade
           end
         end
         # choose the appropriate class
-        klass=  Dry::Core::ClassBuilder.new(  name: type_name, parent: nil, namespace: namespace ).call
+        # First, try to find an existing model class
+        klass = nil
+        begin
+          klass = namespace.const_get(type_name) if namespace && namespace.const_defined?(type_name)
+        rescue NameError
+          # Class not found, will create dynamically
+        end
+        
+        # If no existing class found, create a new one
+        unless klass
+          klass = Dry::Core::ClassBuilder.new( name: type_name, parent: nil, namespace: namespace ).call
+        end
         #
       begin
         # create a new object of that  class with the appropriate attributes