htm 0.0.31 → 0.0.32

data/lib/htm/jobs/generate_embedding_job.rb

@@ -7,70 +7,54 @@ class HTM
   module Jobs
     # Background job to generate and store vector embeddings for nodes
     #
-    # This job is enqueued after a node is saved to avoid blocking the
-    # main request path. It generates embeddings asynchronously and updates
-    # the node record with the embedding vector.
-    #
     # @see ADR-016: Async Embedding and Tag Generation
     #
     class GenerateEmbeddingJob
       # Generate embedding for a node
       #
-      # Uses the configured embedding generator (HTM.embed) which delegates
-      # to the application-provided or default RubyLLM implementation.
-      #
       # @param node_id [Integer] ID of the node to process
       #
       def self.perform(node_id:)
-        node = HTM::Models::Node.first(id: node_id)
-
-        unless node
-          HTM.logger.warn "GenerateEmbeddingJob: Node #{node_id} not found"
-          return
-        end
-
-        # Skip if already has embedding
+        node = find_node(node_id) or return
         return if node.embedding
 
-        provider = HTM.configuration.embedding_provider.to_s
+        provider   = HTM.configuration.embedding_provider.to_s
         start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
         begin
-          # Generate and process embedding using EmbeddingService
           result = HTM::EmbeddingService.generate(node.content)
-
-          # Update node with processed embedding
           node.update(embedding: result[:storage_embedding])
-
-          # Record success metrics
-          elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round
-          HTM::Telemetry.embedding_latency.record(elapsed_ms, attributes: { 'provider' => provider, 'status' => 'success' })
-          HTM::Telemetry.job_counter.add(1, attributes: { 'job' => 'embedding', 'status' => 'success' })
-
-          HTM.logger.info "GenerateEmbeddingJob: Successfully generated embedding for node #{node_id} (#{result[:dimension]} dimensions)"
-
-        rescue HTM::CircuitBreakerOpenError => e
-          # Circuit breaker is open - service is unavailable, will retry later
+          record_telemetry(provider, start_time, 'success', :embedding)
+          HTM.logger.info "GenerateEmbeddingJob: Generated embedding for node #{node_id} (#{result[:dimension]} dimensions)"
+        rescue HTM::CircuitBreakerOpenError
           HTM::Telemetry.job_counter.add(1, attributes: { 'job' => 'embedding', 'status' => 'circuit_open' })
-          HTM.logger.warn "GenerateEmbeddingJob: Circuit breaker open for node #{node_id}, will retry when service recovers"
-
+          HTM.logger.warn "GenerateEmbeddingJob: Circuit breaker open for node #{node_id}"
         rescue HTM::EmbeddingError => e
-          # Record failure metrics
-          elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round
-          HTM::Telemetry.embedding_latency.record(elapsed_ms, attributes: { 'provider' => provider, 'status' => 'error' })
-          HTM::Telemetry.job_counter.add(1, attributes: { 'job' => 'embedding', 'status' => 'error' })
+          record_telemetry(provider, start_time, 'error', :embedding)
+          HTM.logger.error "GenerateEmbeddingJob: Embedding failed for node #{node_id}: #{e.message}"
+        rescue StandardError => e
+          record_telemetry(provider, start_time, 'error', :embedding)
+          HTM.logger.error "GenerateEmbeddingJob: Unexpected error for node #{node_id}: #{e.class.name} - #{e.message}"
+        end
+      end
 
-          # Log embedding-specific errors
-          HTM.logger.error "GenerateEmbeddingJob: Embedding generation failed for node #{node_id}: #{e.message}"
+      class << self
+        private
 
-        rescue StandardError => e
-          # Record failure metrics
-          elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round
-          HTM::Telemetry.embedding_latency.record(elapsed_ms, attributes: { 'provider' => provider, 'status' => 'error' })
-          HTM::Telemetry.job_counter.add(1, attributes: { 'job' => 'embedding', 'status' => 'error' })
+        def find_node(node_id)
+          node = HTM::Models::Node.first(id: node_id)
+          HTM.logger.warn "GenerateEmbeddingJob: Node #{node_id} not found" unless node
+          node
+        end
 
-          # Log unexpected errors
-          HTM.logger.error "GenerateEmbeddingJob: Unexpected error for node #{node_id}: #{e.class.name} - #{e.message}"
+        def elapsed_ms(start_time)
+          ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round
+        end
+
+        def record_telemetry(provider, start_time, status, metric_type)
+          ms = elapsed_ms(start_time)
+          HTM::Telemetry.public_send(:"#{metric_type}_latency").record(ms, attributes: { 'provider' => provider, 'status' => status })
+          HTM::Telemetry.job_counter.add(1, attributes: { 'job' => metric_type.to_s, 'status' => status })
         end
       end
     end

data/lib/htm/jobs/generate_propositions_job.rb

@@ -7,86 +7,60 @@ class HTM
   module Jobs
     # Background job to extract propositions from nodes and create new nodes
     #
-    # This job is enqueued after a node is saved (if proposition extraction is enabled).
-    # It uses LLM to extract atomic factual propositions from node content and
-    # creates new nodes for each proposition. Proposition nodes are marked with
-    # metadata to prevent recursive extraction.
-    #
     # @see PropositionService
     #
     class GeneratePropositionsJob
       # Generate propositions for a node
       #
-      # Uses the configured proposition extractor (HTM.extract_propositions) which
-      # delegates to the application-provided or default RubyLLM implementation.
-      #
       # @param node_id [Integer] ID of the node to process
       # @param robot_id [Integer] ID of the robot that owns this node
       #
       def self.perform(node_id:, robot_id:)
-        node = HTM::Models::Node.first(id: node_id)
-
-        unless node
-          HTM.logger.warn "GeneratePropositionsJob: Node #{node_id} not found"
-          return
-        end
-
-        # Skip if this node is already a proposition (prevent recursion)
+        node = find_node(node_id) or return
         return if node.metadata&.dig('is_proposition')
 
         begin
-          # Extract propositions using PropositionService
           propositions = HTM::PropositionService.extract(node.content)
           return if propositions.empty?
 
           HTM.logger.info "GeneratePropositionsJob: Extracted #{propositions.length} propositions for node #{node_id}"
-
-          # Create a node for each proposition
-          created_count = 0
-          propositions.each do |proposition_text|
-            # Calculate token count
-            token_count = HTM.count_tokens(proposition_text)
-
-            # Create proposition node with is_proposition marker
-            proposition_node = HTM::Models::Node.create(
-              content: proposition_text,
-              token_count: token_count,
-              metadata: { is_proposition: true, source_node_id: node_id }
-            )
-
-            # Link to robot via RobotNode
-            HTM::Models::RobotNode.find_or_create(
-              robot_id: robot_id,
-              node_id: proposition_node.id
-            )
-
-            # Enqueue embedding and tag jobs for the new proposition node
-            # (but NOT another propositions job - the is_proposition marker prevents that)
-            HTM::JobAdapter.enqueue(HTM::Jobs::GenerateEmbeddingJob, node_id: proposition_node.id)
-            HTM::JobAdapter.enqueue(HTM::Jobs::GenerateTagsJob, node_id: proposition_node.id)
-
-            created_count += 1
-          end
-
-          HTM.logger.info "GeneratePropositionsJob: Created #{created_count} proposition nodes from node #{node_id}"
-
+          created = create_proposition_nodes(propositions, source_node_id: node_id, robot_id: robot_id)
+          HTM.logger.info "GeneratePropositionsJob: Created #{created} proposition nodes from node #{node_id}"
         rescue HTM::CircuitBreakerOpenError
-          # Circuit breaker is open - service is unavailable, will retry later
-          HTM.logger.warn "GeneratePropositionsJob: Circuit breaker open for node #{node_id}, will retry when service recovers"
-
+          HTM.logger.warn "GeneratePropositionsJob: Circuit breaker open for node #{node_id}"
         rescue HTM::PropositionError => e
-          # Log proposition-specific errors
           HTM.logger.error "GeneratePropositionsJob: Proposition extraction failed for node #{node_id}: #{e.message}"
-
         rescue Sequel::ValidationFailed => e
-          # Log validation errors
           HTM.logger.error "GeneratePropositionsJob: Database validation failed for node #{node_id}: #{e.message}"
-
         rescue StandardError => e
-          # Log unexpected errors
           HTM.logger.error "GeneratePropositionsJob: Unexpected error for node #{node_id}: #{e.class.name} - #{e.message}"
         end
       end
+
+      class << self
+        private
+
+        def find_node(node_id)
+          node = HTM::Models::Node.first(id: node_id)
+          HTM.logger.warn "GeneratePropositionsJob: Node #{node_id} not found" unless node
+          node
+        end
+
+        def create_proposition_nodes(propositions, source_node_id:, robot_id:)
+          propositions.count do |text|
+            token_count = HTM.count_tokens(text)
+            prop_node = HTM::Models::Node.create(
+              content: text,
+              token_count: token_count,
+              metadata: { is_proposition: true, source_node_id: source_node_id }
+            )
+            HTM::Models::RobotNode.find_or_create(robot_id: robot_id, node_id: prop_node.id)
+            HTM::JobAdapter.enqueue(HTM::Jobs::GenerateEmbeddingJob, node_id: prop_node.id)
+            HTM::JobAdapter.enqueue(HTM::Jobs::GenerateTagsJob, node_id: prop_node.id)
+            true
+          end
+        end
+      end
     end
   end
 end

data/lib/htm/jobs/generate_relationships_job.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+class HTM
+  module Jobs
+    # Background job to compute and upsert weighted edges between nodes.
+    #
+    # Runs after GenerateTagsJob so the node's tags are already present.
+    # For each node, finds all other nodes sharing at least one tag and
+    # computes Jaccard similarity as the edge weight:
+    #
+    #   weight = |tags(A) ∩ tags(B)| / |tags(A) ∪ tags(B)|
+    #
+    # Both directions are stored (A→B and B→A) so the CTE traversal only
+    # needs WHERE source_id IN (seeds) with a plain btree index hit.
+    #
+    # Edges with weight below MIN_WEIGHT_THRESHOLD are skipped.
+    # At most MAX_EDGES_PER_NODE edges are created (highest-weight first).
+    #
+    MIN_WEIGHT_THRESHOLD = 0.1
+    MAX_EDGES_PER_NODE   = 50
+
+    class GenerateRelationshipsJob
+      # Compute and persist relationship edges for a node.
+      #
+      # @param node_id [Integer] ID of the node to process
+      #
+      def self.perform(node_id:)
+        find_node(node_id) or return
+
+        start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+        begin
+          candidates = compute_candidates(node_id)
+
+          if candidates.empty?
+            HTM.logger.info "GenerateRelationshipsJob: No tag-sharing neighbors for node #{node_id}"
+            return
+          end
+
+          count = upsert_edges(node_id, candidates)
+          elapsed = elapsed_ms(start_time)
+          HTM.logger.info "GenerateRelationshipsJob: Upserted #{count} edges for node #{node_id} (#{elapsed}ms)"
+        rescue StandardError => e
+          HTM.logger.error "GenerateRelationshipsJob: Failed for node #{node_id}: #{e.class.name} - #{e.message}"
+        end
+      end
+
+      class << self
+        private
+
+        def find_node(node_id)
+          node = HTM::Models::Node.first(id: node_id)
+          HTM.logger.warn "GenerateRelationshipsJob: Node #{node_id} not found" unless node
+          node
+        end
+
+        # Return candidate neighbor rows [{target_id:, weight:}] sorted by weight desc.
+        # Uses a single SQL query to compute Jaccard similarity for all tag-sharing nodes.
+        #
+        def compute_candidates(node_id)
+          HTM.db.fetch(<<~SQL, node_id, node_id, MAX_EDGES_PER_NODE).all
+            WITH node_a_tags AS (
+              SELECT tag_id
+              FROM node_tags
+              WHERE node_id = ?
+                AND deleted_at IS NULL
+            ),
+            shared AS (
+              SELECT nt.node_id AS target_id, COUNT(*) AS shared_count
+              FROM node_tags nt
+              WHERE nt.tag_id IN (SELECT tag_id FROM node_a_tags)
+                AND nt.node_id != ?
+                AND nt.deleted_at IS NULL
+              GROUP BY nt.node_id
+            ),
+            target_tag_counts AS (
+              SELECT node_id, COUNT(*) AS tag_count
+              FROM node_tags
+              WHERE node_id IN (SELECT target_id FROM shared)
+                AND deleted_at IS NULL
+              GROUP BY node_id
+            ),
+            source_tag_count AS (
+              SELECT COUNT(*) AS tag_count FROM node_a_tags
+            )
+            SELECT
+              s.target_id,
+              s.shared_count::float /
+                (sc.tag_count + tc.tag_count - s.shared_count)::float AS weight
+            FROM shared s
+            JOIN target_tag_counts tc ON tc.node_id = s.target_id
+            CROSS JOIN source_tag_count sc
+            WHERE sc.tag_count > 0
+              AND (sc.tag_count + tc.tag_count - s.shared_count) > 0
+            ORDER BY weight DESC
+            LIMIT ?
+          SQL
+        end
+
+        # Upsert both directions for each candidate above the weight threshold.
+        # Uses INSERT ... ON CONFLICT DO UPDATE so re-runs refresh stale weights.
+        #
+        # @return [Integer] number of edge-pairs inserted or updated
+        #
+        def upsert_edges(node_id, candidates)
+          now  = Time.now
+          rows = []
+
+          candidates.each do |row|
+            weight = row[:weight].to_f
+            next if weight < MIN_WEIGHT_THRESHOLD
+
+            rows << { source_id: node_id,          target_id: row[:target_id],
+                      rel_type: 'related_to', origin: 'tag_cooccurrence',
+                      weight: weight, created_at: now, updated_at: now }
+            rows << { source_id: row[:target_id], target_id: node_id,
+                      rel_type: 'related_to', origin: 'tag_cooccurrence',
+                      weight: weight, created_at: now, updated_at: now }
+          end
+
+          return 0 if rows.empty?
+
+          HTM.db[:node_relationships].insert_conflict(
+            target:  %i[source_id target_id rel_type],
+            update:  { weight: Sequel[:excluded][:weight], updated_at: Sequel[:excluded][:updated_at] }
+          ).multi_insert(rows)
+
+          rows.length / 2
+        end
+
+        def elapsed_ms(start_time)
+          ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round
+        end
+      end
+    end
+  end
+end

data/lib/htm/jobs/generate_tags_job.rb

@@ -7,96 +7,74 @@ class HTM
   module Jobs
     # Background job to generate and associate tags for nodes
     #
-    # This job is enqueued after a node is saved to avoid blocking the
-    # main request path. It uses LLM to extract hierarchical tags from
-    # node content and creates the necessary database associations.
-    #
     # @see ADR-016: Async Embedding and Tag Generation
     # @see ADR-015: Hierarchical Tag Ontology and LLM Extraction
     #
     class GenerateTagsJob
       # Generate tags for a node
       #
-      # Uses the configured tag extractor (HTM.extract_tags) which delegates
-      # to the application-provided or default RubyLLM implementation.
-      #
       # @param node_id [Integer] ID of the node to process
       #
       def self.perform(node_id:)
-        node = HTM::Models::Node.first(id: node_id)
-
-        unless node
-          HTM.logger.warn "GenerateTagsJob: Node #{node_id} not found"
-          return
-        end
+        node = find_node(node_id) or return
 
-        provider = HTM.configuration.tag_provider.to_s
+        provider   = HTM.configuration.tag_provider.to_s
         start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
         begin
-          # Get existing ontology for context (sample of recent tags)
-          existing_ontology = HTM::Models::Tag
-            .order(Sequel.desc(:created_at))
-            .limit(100)
-            .select_map(:name)
-
-          # Extract and validate tags using TagService
-          tag_names = HTM::TagService.extract(node.content, existing_ontology: existing_ontology)
+          tag_names = extract_tags_for(node)
           return if tag_names.empty?
 
-          # Create or find tags (including all parent tags) and associate with node
-          # For "database:postgresql:extensions", this creates and associates:
-          # - "database"
-          # - "database:postgresql"
-          # - "database:postgresql:extensions"
-          tag_names.each do |tag_name|
-            HTM::Models::Tag.find_or_create_with_ancestors(tag_name).each do |tag|
-              # Create association if it doesn't exist
-              HTM::Models::NodeTag.find_or_create(
-                node_id: node.id,
-                tag_id: tag.id
-              )
-            end
-          end
-
-          # Record success metrics
-          elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round
-          HTM::Telemetry.tag_latency.record(elapsed_ms, attributes: { 'provider' => provider, 'status' => 'success' })
-          HTM::Telemetry.job_counter.add(1, attributes: { 'job' => 'tags', 'status' => 'success' })
+          associate_tags(node, tag_names)
+          record_telemetry(provider, start_time, 'success')
+          HTM.logger.info "GenerateTagsJob: Generated #{tag_names.length} tags for node #{node_id}: #{tag_names.join(', ')}"
 
-          HTM.logger.info "GenerateTagsJob: Successfully generated #{tag_names.length} tags for node #{node_id}: #{tag_names.join(', ')}"
-
-        rescue HTM::CircuitBreakerOpenError => e
-          # Circuit breaker is open - service is unavailable, will retry later
+          HTM::JobAdapter.enqueue(HTM::Jobs::GenerateRelationshipsJob, node_id: node_id)
+        rescue HTM::CircuitBreakerOpenError
           HTM::Telemetry.job_counter.add(1, attributes: { 'job' => 'tags', 'status' => 'circuit_open' })
-          HTM.logger.warn "GenerateTagsJob: Circuit breaker open for node #{node_id}, will retry when service recovers"
+          HTM.logger.warn "GenerateTagsJob: Circuit breaker open for node #{node_id}"
+        rescue HTM::TagError, Sequel::ValidationFailed => e
+          record_telemetry(provider, start_time, 'error')
+          HTM.logger.error "GenerateTagsJob: Failed for node #{node_id}: #{e.message}"
+        rescue StandardError => e
+          record_telemetry(provider, start_time, 'error')
+          HTM.logger.error "GenerateTagsJob: Unexpected error for node #{node_id}: #{e.class.name} - #{e.message}"
+        end
+      end
 
-        rescue HTM::TagError => e
-          # Record failure metrics
-          elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round
-          HTM::Telemetry.tag_latency.record(elapsed_ms, attributes: { 'provider' => provider, 'status' => 'error' })
-          HTM::Telemetry.job_counter.add(1, attributes: { 'job' => 'tags', 'status' => 'error' })
+      class << self
+        private
 
-          # Log tag-specific errors
-          HTM.logger.error "GenerateTagsJob: Tag generation failed for node #{node_id}: #{e.message}"
+        def find_node(node_id)
+          node = HTM::Models::Node.first(id: node_id)
+          HTM.logger.warn "GenerateTagsJob: Node #{node_id} not found" unless node
+          node
+        end
 
-        rescue Sequel::ValidationFailed => e
-          # Record failure metrics
-          elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round
-          HTM::Telemetry.tag_latency.record(elapsed_ms, attributes: { 'provider' => provider, 'status' => 'error' })
-          HTM::Telemetry.job_counter.add(1, attributes: { 'job' => 'tags', 'status' => 'error' })
+        def extract_tags_for(node)
+          existing_ontology = HTM::Models::Tag
+                              .order(Sequel.desc(:created_at))
+                              .limit(100)
+                              .select_map(:name)
+          HTM::TagService.extract(node.content, existing_ontology: existing_ontology)
+        end
 
-          # Log validation errors
-          HTM.logger.error "GenerateTagsJob: Database validation failed for node #{node_id}: #{e.message}"
+        def associate_tags(node, tag_names)
+          tag_names.each do |tag_name|
+            HTM::Models::Tag.find_or_create_with_ancestors(tag_name).each do |tag|
+              HTM::Models::NodeTag.find_or_create(node_id: node.id, tag_id: tag.id)
+            end
+          end
+        end
 
-        rescue StandardError => e
-          # Record failure metrics
-          elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round
-          HTM::Telemetry.tag_latency.record(elapsed_ms, attributes: { 'provider' => provider, 'status' => 'error' })
-          HTM::Telemetry.job_counter.add(1, attributes: { 'job' => 'tags', 'status' => 'error' })
+        def elapsed_ms(start_time)
+          ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round
+        end
 
-          # Log unexpected errors
-          HTM.logger.error "GenerateTagsJob: Unexpected error for node #{node_id}: #{e.class.name} - #{e.message}"
+        def record_telemetry(provider, start_time, status)
+          ms = elapsed_ms(start_time)
+          HTM::Telemetry.tag_latency.record(ms, attributes: { 'provider' => provider, 'status' => status })
+          HTM::Telemetry.job_counter.add(1, attributes: { 'job' => 'tags', 'status' => status })
         end
       end
     end