@henrychong-ai/mcp-neo4j-knowledge-graph 2.3.2 → 2.4.1

package/README.md

@@ -595,6 +595,10 @@ WRITE_EMBEDDINGS_LOCALLY=true       # Default true. Set to "false" on thin-clien
 EMBEDDING_BACKFILL_CRON='0 19 * * *' # Cron schedule for scheduleIncrementalRegeneration. Default
                                      # 19:00 UTC daily (= 03:00 SGT). Server-side instances may
                                      # tighten to '*/1 * * * *' for ~1-minute backfill latency.
+EMBEDDING_STALE_CLAIM_MS=300000      # (v2.4.0+) Claims older than this age are auto-released back
+                                     # to 'pending' on the next processJobs tick. Default 5 minutes.
+                                     # Increase if your worker's batch processing time can exceed
+                                     # this; decrease for faster recovery from worker crashes.
 
 # Logging Configuration
 LOG_LEVEL=warn              # Log level: debug, info, warn, error, silent (default: warn)

package/dist/embeddings/EmbeddingJobManager.d.ts

@@ -3,49 +3,32 @@ import type { Entity } from '../KnowledgeGraphManager.js';
 import type { StorageProvider } from '../storage/StorageProvider.js';
 import type { EntityEmbedding } from '../types/entity-embedding.js';
 import type { EmbeddingService } from './EmbeddingService.js';
-/**
- * Interface for embedding cache options
- */
+import type { JobStore } from './JobStore.js';
 interface CacheOptions {
     size: number;
     ttl: number;
     maxItems?: number;
     ttlHours?: number;
 }
-/**
- * Interface for rate limiting options
- */
 interface RateLimiterOptions {
     tokensPerInterval: number;
     interval: number;
 }
-/**
- * Interface for job processing results
- */
 interface JobProcessResults {
     processed: number;
     successful: number;
     failed: number;
 }
-/**
- * Interface for the rate limiter status
- */
 interface RateLimiterStatus {
     availableTokens: number;
     maxTokens: number;
     resetInMs: number;
 }
-/**
- * Interface for a cached embedding entry
- */
 interface CachedEmbedding {
     embedding: number[];
     timestamp: number;
     model: string;
 }
-/**
- * Interface for a logger
- */
 interface Logger {
     debug: (message: string, meta?: Record<string, unknown>) => void;
     info: (message: string, meta?: Record<string, unknown>) => void;
@@ -53,25 +36,20 @@ interface Logger {
     error: (message: string, meta?: Record<string, unknown>) => void;
 }
 /**
- * Interface for embedding storage provider, extending the base provider
+ * Storage shape required by `EmbeddingJobManager` for entity access.
+ * Persistence of the queue itself is delegated to `JobStore`.
  */
 interface EmbeddingStorageProvider extends StorageProvider {
-    /**
-     * Access to the underlying database
-     */
-    db: any;
-    /**
-     * Get an entity by name
-     */
     getEntity(entityName: string): Promise<Entity | null>;
+    storeEntityVector(entityName: string, embedding: EntityEmbedding): Promise<void>;
     /**
-     * Store an entity vector embedding
+     * Optional: efficient predicate-based lookup for entities lacking an embedding.
+     * When present, `scheduleIncrementalRegeneration` prefers it over the
+     * `loadGraph()`-and-filter fallback (which strips the embedding property in
+     * its return mapper and so always reports 100% missing).
      */
-    storeEntityVector(entityName: string, embedding: EntityEmbedding): Promise<void>;
+    getEntityNamesMissingEmbeddings?(): Promise<string[]>;
 }
-/**
- * Return structure for queue status
- */
 interface QueueStatus {
     pending: number;
     processing: number;
@@ -80,7 +58,11 @@ interface QueueStatus {
     totalJobs: number;
 }
 /**
- * Manages embedding jobs for semantic search
+ * Manages embedding jobs for semantic search.
+ *
+ * Persistence of the queue lives behind a `JobStore` — production wiring
+ * uses `Neo4jJobStore`, which stores jobs as `:EmbeddingJob` nodes and
+ * makes `claim()` safe under multi-worker contention.
  */
 export declare class EmbeddingJobManager {
     private storageProvider;
@@ -94,134 +76,47 @@ export declare class EmbeddingJobManager {
     cache: LRUCache<string, CachedEmbedding>;
     private cacheOptions;
     private logger;
-    /**
-     * Creates a new embedding job manager
-     *
-     * @param storageProvider - Provider for entity storage
-     * @param embeddingService - Service to generate embeddings
-     * @param rateLimiterOptions - Optional configuration for rate limiting
-     * @param cacheOptions - Optional configuration for caching
-     * @param logger - Optional logger for operation logging
-     */
-    constructor(storageProvider: EmbeddingStorageProvider, embeddingService: EmbeddingService, rateLimiterOptions?: RateLimiterOptions | null, cacheOptions?: CacheOptions | null, logger?: Logger | null);
-    /**
-     * Initialize the database schema for embedding jobs
-     *
-     * @private
-     */
-    private _initializeDatabase;
-    /**
-     * Schedule an entity for embedding generation
-     *
-     * @param entityName - Name of the entity to generate embedding for
-     * @param priority - Optional priority (higher priority jobs are processed first)
-     * @returns Job ID
-     */
+    private jobStore;
+    private staleClaimMs;
+    /** Stable id for this process — visible in `:EmbeddingJob.claimedBy`. */
+    readonly workerId: string;
+    constructor(storageProvider: EmbeddingStorageProvider, embeddingService: EmbeddingService, rateLimiterOptions?: RateLimiterOptions | null, cacheOptions?: CacheOptions | null, logger?: Logger | null, jobStore?: JobStore, staleClaimMs?: number);
     scheduleEntityEmbedding(entityName: string, priority?: number): Promise<string>;
-    /**
-     * Process a batch of pending embedding jobs
-     *
-     * @param batchSize - Maximum number of jobs to process
-     * @returns Result statistics
-     */
     processJobs(batchSize?: number): Promise<JobProcessResults>;
-    /**
-     * Get the current status of the job queue
-     *
-     * @returns Queue statistics
-     */
     getQueueStatus(): Promise<QueueStatus>;
-    /**
-     * Retry failed embedding jobs
-     *
-     * @returns Number of jobs reset for retry
-     */
     retryFailedJobs(): Promise<number>;
-    /**
-     * Clean up old completed jobs
-     *
-     * @param threshold - Age in milliseconds after which to delete completed jobs, defaults to 7 days
-     * @returns Number of jobs cleaned up
-     */
     cleanupJobs(threshold?: number): Promise<number>;
     /**
-     * Update a job's status in the database
-     *
-     * @private
-     * @param jobId - ID of the job to update
-     * @param status - New status
-     * @param attempts - Optional attempts count update
-     * @param error - Optional error message
-     * @returns Database result
-     */
-    private _updateJobStatus;
-    /**
-     * Check rate limiter and consume a token if available
-     *
-     * @private
-     * @returns Object with success flag
+     * Token-bucket rate limiter. Public for legacy test compatibility — was
+     * `_checkRateLimiter` historically; kept callable from tests via underscore.
      */
     _checkRateLimiter(): {
         success: boolean;
     };
-    /**
-     * Get the current status of the rate limiter
-     *
-     * @returns Rate limiter status information
-     */
     getRateLimiterStatus(): RateLimiterStatus;
-    /**
-     * Retrieve a cached embedding or generate a new one
-     *
-     * @param text - Text to generate embedding for
-     * @returns Embedding vector
-     */
     _getCachedEmbeddingOrGenerate(text: string): Promise<number[]>;
-    /**
-     * Store an embedding in the cache
-     *
-     * @private
-     * @param text - Original text
-     * @param embedding - Embedding vector
-     */
     private _cacheEmbedding;
-    /**
-     * Generate a deterministic cache key for text
-     *
-     * @private
-     * @param text - Text to hash
-     * @returns Cache key
-     */
     _generateCacheKey(text: string): string;
-    /**
-     * Prepare text for embedding from an entity
-     *
-     * @private
-     * @param entity - Entity to prepare text from
-     * @returns Processed text ready for embedding
-     */
     private _prepareEntityText;
-    /**
-     * Get a cached embedding entry (used for testing)
-     *
-     * @param key - Cache key
-     * @returns Cached embedding or undefined
-     */
     getCacheEntry(key: string): CachedEmbedding | undefined;
     /**
-     * Schedule incremental regeneration for entities without embeddings
-     * This method queries all entities and schedules embedding jobs only for those missing embeddings
+     * Enqueue embedding jobs for every currently-valid entity that lacks an
+     * embedding. Intended for a server-side cron tick to backfill entities
+     * created by thin clients running with `WRITE_EMBEDDINGS_LOCALLY=false`.
      *
-     * @returns Number of entities scheduled for embedding generation
+     * v2.4.1+ — prefers `storageProvider.getEntityNamesMissingEmbeddings()`
+     * when available (a single Cypher predicate, no client-side filtering).
+     * Falls back to the legacy `loadGraph`-and-filter path for compatibility,
+     * but that path is buggy with the current `nodeToEntity` mapper which
+     * strips the embedding property — so it always reports 100% missing.
      */
     scheduleIncrementalRegeneration(): Promise<number>;
     /**
-     * Get all entities from storage provider
-     * This is a helper method to retrieve all entities for incremental regeneration
-     *
-     * @private
-     * @returns Array of all entities
+     * Resolve the list of entity names lacking embeddings. Prefer the storage
+     * provider's dedicated Cypher predicate; otherwise fall back to walking
+     * `loadGraph()` (legacy, suboptimal — see the v2.4.1 notes above).
      */
+    private _getEntityNamesMissingEmbeddings;
     private _getAllEntitiesFromStorage;
 }
 export {};