arcadedb 0.5.0 → 0.5.4

data/examples/rag_usecase_demo.rb

@@ -0,0 +1,345 @@
+# RAG Use Case Demo - Knowledge Base Search
+#
+# This example demonstrates a complete RAG (Retrieval-Augmented Generation) workflow
+# using a knowledge base of technical documentation.
+#
+# Features demonstrated:
+# - Document ingestion with vector embeddings
+# - Vector similarity search for semantic queries
+# - Hybrid search combining vector + full-text
+# - Category and tag-based filtering
+# - Real-world query scenarios
+#
+# Run with: ruby examples/rag_usecase_demo.rb
+
+require 'bundler/setup'
+require 'arcade'
+
+# =============================================================================
+# Model Definition
+# =============================================================================
+
+module UseCase
+  class KnowledgeDocument < Arcade::Vertex
+    include Arcade::Ragschema
+
+    attribute :title, Types::String
+    attribute :content, Types::String
+    attribute :embedding?, Types::Array.of(Types::Float).optional
+    attribute :category, Types::String.optional
+    attribute :tags, Types::Array.of(Types::String).optional
+    attribute :author, Types::String.optional
+    attribute :source_url, Types::String.optional
+    attribute :created_at, Types::DateTime.optional
+    attribute :updated_at, Types::DateTime.optional
+
+    def self.db_init
+      rag_db_init(
+        vector_dimension: 1536,
+        full_text_fields: ['title', 'content', 'category']
+      )
+    end
+  end
+
+  # =============================================================================
+  # Helper for generating mock embeddings (for demo purposes)
+  # In production, use a real embedding service (OpenAI, HuggingFace, etc.)
+  # =============================================================================
+
+  module EmbeddingHelper
+    ##
+    # Generate a deterministic mock embedding for demonstration
+    #
+    # @param text [String] Text to embed
+    # @param seed [Integer] Seed for deterministic generation
+    # @return [Array<Float>] 1536-dimensional vector
+    def self.generate_mock_embedding(text, seed: 42)
+      base_hash = (text + seed.to_s).hash
+      vector = []
+      1536.times do |i|
+        value = ((base_hash * (i + 1) * 31) % 10000) / 10000.0 - 0.5
+        vector << value
+      end
+      vector
+    end
+  end
+end
+
+# =============================================================================
+# Demo Data - Technical Knowledge Base
+# =============================================================================
+
+KNOWLEDGE_BASE = [
+  {
+    title: 'Introduction to Graph Databases',
+    content: 'Graph databases are designed to store and treat relationships between data entities as first-class citizens. Unlike relational databases, graph databases use nodes and edges to represent and store data. This makes them ideal for analyzing complex relationships and patterns in data.',
+    category: 'Database Fundamentals',
+    tags: ['graph', 'database', 'nosql', 'fundamentals'],
+    author: 'Database Team'
+  },
+  {
+    title: 'ArcadeDB Overview',
+    content: 'ArcadeDB is a multi-model graph database built from the ground up with support for vertices, edges, documents, arrays, and vectors. It provides a SQL-like query language and supports RAG (Retrieval-Augmented Generation) workloads with native vector search capabilities.',
+    category: 'Database Systems',
+    tags: ['arcadedb', 'graph', 'vector', 'rag'],
+    author: 'ArcadeDB Team'
+  },
+  {
+    title: 'Vector Search Fundamentals',
+    content: 'Vector search, also known as semantic search, uses mathematical representations of data called embeddings. These embeddings capture the semantic meaning of text, allowing for similarity-based searches that understand context rather than just keyword matching.',
+    category: 'Search Technology',
+    tags: ['vector', 'embedding', 'similarity', 'ai'],
+    author: 'AI Research Team'
+  },
+  {
+    title: 'RAG Architecture Patterns',
+    content: 'Retrieval-Augmented Generation (RAG) combines retrieval-based and generation-based approaches. It first retrieves relevant documents from a knowledge base using vector similarity, then uses a language model to generate responses based on the retrieved context.',
+    category: 'AI Architecture',
+    tags: ['rag', 'llm', 'retrieval', 'generation'],
+    author: 'ML Engineering'
+  },
+  {
+    title: 'SQL Query Optimization',
+    content: 'Query optimization in SQL databases involves techniques like indexing, query planning, and execution strategy selection. Proper indexing strategies can dramatically improve query performance, especially for complex joins and aggregations.',
+    category: 'Database Performance',
+    tags: ['sql', 'optimization', 'indexing', 'performance'],
+    author: 'Performance Team'
+  },
+  {
+    title: 'NoSQL vs SQL Databases',
+    content: 'NoSQL databases offer flexible schemas and horizontal scaling, making them suitable for unstructured data and high-velocity applications. SQL databases provide ACID transactions and structured querying capabilities. The choice depends on your specific use case requirements.',
+    category: 'Database Comparison',
+    tags: ['nosql', 'sql', 'comparison', 'architecture'],
+    author: 'Architecture Team'
+  },
+  {
+    title: 'Full-Text Search Implementation',
+    content: 'Full-text search indexes allow for efficient text querying beyond simple LIKE matches. They support tokenization, stemming, and relevance scoring. Modern search engines combine full-text search with vector embeddings for hybrid search capabilities.',
+    category: 'Search Technology',
+    tags: ['fulltext', 'search', 'indexing', 'hybrid'],
+    author: 'Search Team'
+  },
+  {
+    title: 'Embedding Models for Text',
+    content: 'Text embedding models like BERT, Sentence Transformers, and OpenAI embeddings convert text into dense vectors. These vectors capture semantic relationships, enabling tasks like semantic search, clustering, and similarity detection.',
+    category: 'Machine Learning',
+    tags: ['embedding', 'bert', 'nlp', 'transformers'],
+    author: 'NLP Team'
+  }
+].freeze
+
+# =============================================================================
+# Demo Functions
+# =============================================================================
+
+def print_header(title)
+  puts "\n" + '=' * 70
+  puts title.center(70)
+  puts '=' * 70
+end
+
+def print_section(title)
+  puts "\n--- #{title} ---\n"
+end
+
+def print_document(doc, show_similarity: false)
+  puts "  Title: #{doc.title}"
+  puts "  Category: #{doc.category}"
+  puts "  Author: #{doc.author}"
+  puts "  Tags: #{doc.tags.join(', ')}"
+  puts "  Content: #{doc.content[0..150]}..."
+  puts "  Similarity: #{doc.similarity.round(4)}" if doc.respond_to?(:similarity) && doc.similarity
+  puts
+end
+
+def load_knowledge_base
+  print_section('Loading Knowledge Base')
+
+  KNOWLEDGE_BASE.each do |doc_data|
+    embedding = UseCase::EmbeddingHelper.generate_mock_embedding(
+      doc_data[:title] + ' ' + doc_data[:content]
+    )
+
+    UseCase::KnowledgeDocument.create(
+      title: doc_data[:title],
+      content: doc_data[:content],
+      embedding: embedding,
+      category: doc_data[:category],
+      tags: doc_data[:tags],
+      author: doc_data[:author],
+      created_at: DateTime.now
+    )
+
+    puts "  ✓ Loaded: #{doc_data[:title]}"
+  end
+
+  total = UseCase::KnowledgeDocument.count
+  puts "\nTotal documents: #{total}"
+end
+
+def demo_vector_search
+  print_section('Vector Similarity Search')
+
+  queries = [
+    'What is a graph database and how does it work?',
+    'How does RAG architecture work?',
+    'Explain vector embeddings for text search'
+  ]
+
+  queries.each do |query|
+    puts "\nQuery: \"#{query}\""
+    puts "-" * 50
+
+    query_embedding = UseCase::EmbeddingHelper.generate_mock_embedding(query)
+
+    results = UseCase::KnowledgeDocument.vector_search(
+      query_embedding,
+      limit: 3,
+      threshold: 0.0
+    )
+
+    results.each_with_index do |doc, i|
+      puts "  [#{i + 1}] #{doc.title} (similarity: #{doc.similarity.round(4) if doc.similarity})"
+    end
+  end
+end
+
+def demo_hybrid_search
+  print_section('Hybrid Search (Vector + Full-Text)')
+
+  searches = [
+    { query: 'database performance', text: 'optimization' },
+    { query: 'search technology', text: 'Full-Text' },
+    { query: 'machine learning', text: 'embedding' }
+  ]
+
+  searches.each do |search|
+    puts "\nVector Query: \"#{search[:query]}\""
+    puts "Text Filter: \"#{search[:text]}\""
+    puts "-" * 50
+
+    query_embedding = UseCase::EmbeddingHelper.generate_mock_embedding(search[:query])
+
+    results = UseCase::KnowledgeDocument.hybrid_search(
+      query_embedding,
+      search[:text],
+      limit: 5,
+      vector_threshold: 0.0
+    )
+
+    if results.empty?
+      puts "  No results found"
+    else
+      results.each_with_index do |doc, i|
+        puts "  [#{i + 1}] #{doc.title}"
+      end
+    end
+  end
+end
+
+def demo_category_filtering
+  print_section('Category-Based Filtering')
+
+  categories = ['Search Technology', 'Database Systems', 'Machine Learning']
+
+  categories.each do |category|
+    puts "\nCategory: #{category}"
+    puts "-" * 50
+
+    results = UseCase::KnowledgeDocument.where("category = '#{category}'")
+
+    results.each do |doc|
+      puts "  • #{doc.title} (#{doc.author})"
+    end
+  end
+end
+
+def demo_tag_search
+  print_section('Tag-Based Search')
+
+  tags = ['vector', 'graph', 'embedding']
+
+  tags.each do |tag|
+    puts "\nTag: #{tag}"
+    puts "-" * 50
+
+    results = UseCase::KnowledgeDocument.where("tags CONTAINS '#{tag}'")
+
+    results.each do |doc|
+      puts "  • #{doc.title} [#{doc.tags.join(', ')}]"
+    end
+  end
+end
+
+def demo_real_world_queries
+  print_section('Real-World Query Scenarios')
+
+  scenarios = {
+    'What is ArcadeDB?' => 'Definition query',
+    'difference between SQL and NoSQL' => 'Comparison query',
+    'How to optimize database queries?' => 'How-to query',
+    'Explain RAG benefits' => 'Concept exploration'
+  }
+
+  scenarios.each do |query, type|
+    puts "\n[#{type}] \"#{query}\""
+    puts "-" * 50
+
+    query_embedding = UseCase::EmbeddingHelper.generate_mock_embedding(query)
+
+    results = UseCase::KnowledgeDocument.vector_search(
+      query_embedding,
+      limit: 2,
+      threshold: 0.0
+    )
+
+    results.each do |doc|
+      puts "  → #{doc.title}"
+    end
+  end
+end
+
+# =============================================================================
+# Main Demo Runner
+# =============================================================================
+
+def run_demo
+  print_header('RAG Use Case Demo - Knowledge Base Search')
+
+  # Connect to database
+  Arcade::Init.connect('test')
+
+  # Create type if needed
+  unless Arcade::Init.db.types.include?('UseCase_KnowledgeDocument')
+    puts 'Creating KnowledgeDocument type...'
+    UseCase::KnowledgeDocument.create_type
+  end
+
+  # Clear existing data for fresh demo
+  UseCase::KnowledgeDocument.delete_all rescue nil
+
+  # Run demos
+  load_knowledge_base
+  demo_vector_search
+  demo_hybrid_search
+  demo_category_filtering
+  demo_tag_search
+  demo_real_world_queries
+
+  print_header('Demo Complete!')
+  puts "\nThis demo showcased:"
+  puts "  ✓ Document ingestion with vector embeddings"
+  puts "  ✓ Vector similarity search for semantic queries"
+  puts "  ✓ Hybrid search (vector + full-text)"
+  puts "  ✓ Category and tag-based filtering"
+  puts "  ✓ Real-world query scenarios"
+  puts "\nFor more information, see:"
+  puts "  - RAG_SCHEMA.md"
+  puts "  - spec/lib/usecase/rag_integration_spec.rb"
+  puts
+end
+
+# Run if executed directly
+if $0 == __FILE__
+  run_demo
+end

data/lib/arcade/api/operations.rb

@@ -32,6 +32,13 @@ module Arcade
       get_data 'databases'
     end
 
+    # ------------------------------ database?        ------------------------------------------------- #
+    #  returns true if the database exists                                                              #
+
+    def self.database? name
+      get_data "exists/#{name}"
+    end
+
     # ------------------------------ create database  ------------------------------------------------- #
     #  creates a database if not present                                                                #
     def self.create_database name
@@ -54,23 +61,6 @@ module Arcade
       logger.fatal "Drop database #{name} through \"POST drop database/#{name}\" failed"
       raise
     end
-    # ------------------------------  create document ------------------------------------------------- #
-    # adds a document to the specified database table
-    #
-    # specify database-fields as hash-type parameters
-    #
-    # i.e   Arcade::Api.create_document 'devel', 'documents',  name: 'herta meyer', age: 56, sex: 'f'
-    #
-    # returns the rid of the inserted dataset
-    #
-    def self.create_document database, type, session_id: nil, **attributes
-      payload = { "@type" => type }.merge( attributes )
-      if session_id.nil?
-        post_data "document/#{database}", payload
-      else
-        post_transaction "document/#{database}", payload, session_id: session_id
-      end
-    end
 
     # ------------------------------  execute         ------------------------------------------------- #
     # executes a sql-query in the specified database
@@ -94,13 +84,28 @@ module Arcade
     # ------------------------------  query           ------------------------------------------------- #
     # same for idempotent queries
     def self.query database, query, session_id: nil
-      if session_id.nil?
-        post_data   "query/#{database}" , provide_payload(query)
-      else
-        post_transaction "query/#{database}" , provide_payload(query), session_id: session_id
-      end
+#      if session_id.nil?
+        get_data   "query/#{database}/sql/" + provide_payload(query, action: :get)[:query]
+ #     else
+ #       post_transaction "query/#{database}" , provide_payload(query), session_id: session_id
+ #     end
     end
 
+    # ------------------------------  create Document   ------------------------------------------------- #
+    # adds a record ( document, vertex or edge ) to the specified database type
+    #
+    # specify database-fields as hash-type parameters
+    #
+    # i.e   Arcade::Api.create_record 'devel', 'documents',  name: 'herta meyer', age: 56, sex: 'f'
+    #
+    # returns the rid of the inserted dataset
+    #
+    def self.create_document database, type, session_id: nil, **attributes
+        content = "CONTENT #{ attributes.to_json }"
+        result = execute( database, session_id: session_id ){ "INSERT INTO #{type} #{content} "}
+        result.first[:@rid]   #  emulate the »old« behavior of the /post/create_document api endpoint
+    end
+    
     # ------------------------------  get_record      ------------------------------------------------- #
     # fetches a record by providing  database and  rid
     # and returns the result as hash
@@ -114,7 +119,7 @@ module Arcade
       rid =  rid.join(':')
       rid = rid[1..-1] if rid[0]=="#"
       if rid.rid?
-      get_data  "document/#{database}/#{rid}"
+        get_data( "query/#{database}/sql/" + URI.encode_uri_component("select from #{rid}")) &.first
       else
         raise Error "Get requires a rid input"
       end
@@ -181,7 +186,11 @@ module Arcade
                                 the_yield.map do | key,  value |
                                   case key
                                   when :query
-                                   action == :post ? [ :command, value ] : [ :query, value ]
+                                    if action == :post
+                                      [ :command, value ]
+                                    else
+                                      [ :query, URI.encode_uri_component(value )]
+                                    end
                                   when :limit
                                     [ :limit , value ]
                                   when :params

data/lib/arcade/api/primitives.rb

@@ -9,6 +9,12 @@ module Arcade
 
     def http
 #     break_on = -> (response) { response.status == 500  }
+      #     Version 23.12: Persistent connection are inactive after 3 sec.
+      #     Implemented a walk around, that renews the connection after 2 sec. of inactivity
+      t = Time.now
+      @t ||= Time.now
+      @http = nil if t-@t > 2
+      @t = t
       @http ||= HTTPX.plugin(:basic_auth).basic_auth(auth[:username], auth[:password])
                   .plugin(:persistent)
                   .plugin(:circuit_breaker)
@@ -49,7 +55,7 @@ module Arcade
 
     # ------------------------------ post data -------------------------------------------------------- #
     def post_data command,  payload
-      case response = http.post( Config.base_uri + command, json:  payload )
+        case response = http.post( Config.base_uri + command, json:  payload )
         in {status: 200..299}
           # success
         JSON.parse( response.body, symbolize_names: true )[:result]
@@ -64,7 +70,7 @@ module Arcade
              sleep 1
              post_data command,  payload
            else
-           raise Arcade::QueryError.new **response.json( symbolize_names: true  )
+             raise Arcade::QueryError.new **response.json( symbolize_names: true  )
            end
       else
         #   # http error
@@ -73,6 +79,49 @@ module Arcade
   #    response.raise_for_status
     end
 
+    # ------------------------------  post text ----------------------------------------------------- #
+    # posts a plain-text body (used by the TimeSeries Line Protocol endpoint)
+    def post_text command, body
+      case response = http.post( Config.base_uri + command, body: body )
+      in {status: 200..299}
+        response.status
+       in {status: 400..}
+        answer = response.json( symbolize_names: true )
+        answer[:detail] ||= answer[:error]   # ts-endpoints report error only
+        raise Arcade::QueryError.new **answer
+      else
+        raise response
+      end
+    end
+
+    # ------------------------------  post json raw ------------------------------------------------- #
+    # posts a json body, returns the parsed body (not the :result part)
+    def post_json command, payload
+      case response = http.post( Config.base_uri + command, json: payload )
+      in {status: 200..299}
+        JSON.parse( response.body, symbolize_names: true )
+       in {status: 400..}
+        answer = response.json( symbolize_names: true )
+        answer[:detail] ||= answer[:error]
+        raise Arcade::QueryError.new **answer
+      else
+        raise response
+      end
+    end
+
+    # ------------------------------  get json ------------------------------------------------------ #
+    # GET returning the parsed body (not only the :result part)
+    def get_json command
+      case response = http.get( Config.base_uri + command )
+      in {status: 200..299}
+        JSON.parse( response.body, symbolize_names: true )
+       in {status: 400..}
+        raise Arcade::QueryError.new **response.json( symbolize_names: true )
+      else
+        raise response
+      end
+    end
+
     # ------------------------------ transaction      ------------------------------------------------- #
     #
     # The payload, optional as a JSON, accepts the following parameters:
@@ -87,6 +136,7 @@ module Arcade
     # ------------------------------ post transaction ------------------------------------------------- #
     def post_transaction command, params, session_id:
       http_a = http.with(  headers: { "arcadedb-session-id" => session_id } , debug_level: 1)
+      puts "params #{params.inspect}"
       case response = http_a.post( Config.base_uri + command, json:  params )
         in {status: 200..299}
           # success

data/lib/arcade/base.rb

@@ -1,4 +1,5 @@
 module Arcade
+
   class Base  < Dry::Struct
 
     extend Arcade::Support::Sql
@@ -20,6 +21,9 @@ module Arcade
     ## ----------------------------------------- Class Methods------------------------------------ ##
     #                                                                                               #
     class << self
+     def descendants
+         ObjectSpace.each_object(Class).select { |klass| klass < self }
+     end
 
       # this has to be implemented on class level
       # otherwise it interfere  with attributes
@@ -49,7 +53,7 @@ module Arcade
         begin
         loop do
           if the_class.respond_to?(:demodulize)
-            if [ 'Document','Vertex', 'Edge'].include?(the_class.demodulize)
+            if [ 'Document','Vertex', 'Edge', 'Timeseries'].include?(the_class.demodulize)
               if  the_class == superclass  # no inheritance
                   db.create_type the_class.demodulize, to_s.snake_case
               else
@@ -294,16 +298,19 @@ module Arcade
         Query.new( **{ from: self }.merge(args) )
       end
 
-      # immutable support
-      # to make a database type immutable add
+      # ## Immutable Support
+      #
+      # To make a database type immutable add  
       #  `not_permitted :update, :upsert, :delete`
       # to the model-specification
       #
+      # Even after applying `not_permitted` the database-type can be modified via class-methods.
+      #
       def not_permitted *m
         m.each do | def_m |
-          define_method(  def_m ) do | v = nil |
-          raise ArcadeImmutableError "operation not permitted", caller
-        end
+          define_method( def_m ) do | v = nil |
+            raise  Arcade::ImmutableError.new( "operation  #{def_m} not permitted" )
+          end
         end
       end
 
@@ -366,8 +373,6 @@ module Arcade
     end
 
     def to_human
-
-
 		"<#{ self.class.to_s.snake_case }" + rid? ? "[#{ rid }]: " : " " + invariant_attributes.map do |attr, value|
 			v= case value
 				 when Base
@@ -428,7 +433,7 @@ module Arcade
 
     def update **args
       Query.new( from: rid , kind: :update, set: args).execute
-      refresh
+      refresh   # return the updated record (the object itself is untouched!)
     end
 
     # inserts or updates a embedded document
@@ -448,21 +453,37 @@ module Arcade
       db.transmit { " update #{rid} set `#{embedded}`.`#{embedded_property}` =  #{value.to_or}" }
     end
 
-    def update_list list, value
-      value = if value.is_a? Document
+    # Adds List-Elements to embedded List
+    #
+    # Arguments:
+    # * list: A symbol of the list property
+    # * value: A embedded document or a hash
+    # * modus: :auto, :first, :append
+    #
+    # Prefered modus operandi
+    # * the-element.insert  (...) , #{list}:[]
+    # * the_element.update_list list, value: :append
+    #
+    def update_list list, value, modus: :auto
+      value = if value.is_a? Document   # embedded mode
                 value.to_json
               else
                 value.to_or
               end
-      if send( list ).nil? || send( list ).empty?
-        db.transmit { "update #{ rid } set  #{ list } =  [#{ value }]" }
+      if modus == :auto
+        modus =  db.query( "select #{list}.size() from #{rid}" ).select_result.first.zero? ? :first : :append
+      end
+
+     if modus == :first
+        db.transmit { "update #{ rid } set  #{ list } = [#{ value }]" }
       else
         db.transmit { "update #{ rid } set  #{ list } += #{ value }" }
       end
-      refresh
+     # refresh
     end
 
     # updates a map  property ,  actually adds the key-value pair to the property
+    ## does not work on reduced model records
     def update_map m, key, value
       if send( m ).nil?
         db.transmit { "update #{ rid } set #{ m } = MAP ( #{ key.to_s.to_or } , #{ value.to_or } ) "  }

data/lib/arcade/database.rb

@@ -157,7 +157,7 @@ module Arcade
 
 
 
-    # ------------ create  -----------
+    # ------------ create  -----------  ## not supported anymore  by the api
     # returns an rid of the successfully  created vertex or document
     #
     #  Parameter:  name of the vertex or document type
@@ -186,19 +186,28 @@ module Arcade
 
       content_params = params.except( :type, :bucket, :index, :from, :return, :session_id )
       target_params = params.slice( :type, :bucket, :index )
-#      session_id = params[:session_id]   # extraxt session_id  --> future-use? 
+#      session_id = params[:session_id]   # extraxt session_id  --> future-use?
       if  target_params.empty?
         raise "Could not insert: target missing (type:, bucket:, index:)"
       elsif content_params.empty?
         logger.error "Nothing to Insert"
       else
-        content =  "CONTENT #{ content_params.to_json }"
+        content = "CONTENT #{ content_params.to_json }"
         target =  target_params.map{|y,z|  y==:type ?  z : "#{y.to_s} #{ z } "}.join
-        result = Api.execute( database, session_id: session ){ "INSERT INTO #{target} #{content} "}
-        result &.first.allocate_model(false)
+        result =  Api.execute( database, session_id: session ){ "INSERT INTO #{target} #{content} "}
+        # 
+        result &.record&.allocate_model(false)
+        ## commented for now. reactivate if necessary (performance issue)
+        #record = result&.first
+        # Ensure @type is present for proper model allocation
+        #if record && !record.key?(:@type) && target_params[:type]
+        #  record = record.merge( :"@type" => target_params[:type] )
+        #end
+        #record&.allocate_model(false)
       end
     end
 
+
     # ------------------------------  get        ------------------------------------------------------ #
     # Get fetches the record associated with the rid given as parameter.
     #
@@ -224,7 +233,7 @@ module Arcade
       end
     end
 
-    # ------------------------------  get        ------------------------------------------------------ #
+    # ------------------------------  delete     ------------------------------------------------------ #
     #
     #  Delete the specified rid
     #
@@ -272,7 +281,7 @@ module Arcade
      end
     rescue  Dry::Struct::Error, Arcade::QueryError => e
       Api.rollback database, session_id: s, log: false
-      logger.info "Execution  FAILED -->  Status #{e.status}"
+      logger.fatal "Execution  FAILED -->  Status #{e}"
       []  #  return empty result
     end