@semiont/make-meaning 0.4.11 → 0.4.13

package/README.md

@@ -12,8 +12,9 @@ This package implements the actor model from [ARCHITECTURE.md](../../docs/ARCHIT
 
 - **Stower** (write) — the single write gateway to the Knowledge Base; handles all resource and annotation mutations and job lifecycle events
 - **Browser** (read) — handles all KB read queries: resources, annotations, events, annotation history, referenced-by lookups, entity type listing, and directory browse (merging filesystem listings with KB metadata)
-- **Gatherer** (context assembly) — assembles gathered context for annotations (`gather:requested`) and resources (`gather:resource-requested`)
+- **Gatherer** (context assembly) — assembles gathered context for annotations (`gather:requested`) and resources (`gather:resource-requested`); searches vectors for semantically similar passages (adds `semanticContext` to `GatheredContext`)
 - **Matcher** (search/link) — context-driven candidate search with multi-source retrieval, composite structural scoring, and optional LLM semantic scoring
+- **Smelter** (embed) — subscribes to resource/annotation events, chunks text, embeds via `@semiont/vectors`, persists `embedding:computed` events, and indexes into vector store (Qdrant)
 - **CloneTokenManager** (yield) — manages clone token lifecycle for resource cloning
 
 All actors subscribe to the EventBus via RxJS pipelines. They expose only `initialize()` and `stop()` — no public business methods. Callers communicate with actors by putting events on the bus.
@@ -43,7 +44,7 @@ const makeMeaning = await startMakeMeaning(project, config, eventBus, logger);
 
 // Access components
 const { knowledgeSystem, jobQueue } = makeMeaning;
-const { kb, stower, browser, gatherer, matcher, cloneTokenManager } = knowledgeSystem;
+const { kb, stower, browser, gatherer, matcher, smelter, cloneTokenManager } = knowledgeSystem;
 
 // Graceful shutdown
 await makeMeaning.stop();
@@ -51,11 +52,12 @@ await makeMeaning.stop();
 
 This single call initializes:
 - **KnowledgeSystem** — groups the Knowledge Base and its actors
-  - **KnowledgeBase** — groups EventStore, ViewStorage, WorkingTreeStore, GraphDatabase, and GraphDBConsumer
+  - **KnowledgeBase** — groups EventStore, ViewStorage, WorkingTreeStore, GraphDatabase, GraphDBConsumer, and optionally VectorStore and Smelter
   - **Stower** — subscribes to write commands on EventBus
   - **Browser** — subscribes to all KB read queries and directory browse requests on EventBus
-  - **Gatherer** — subscribes to annotation and resource gather requests on EventBus
+  - **Gatherer** — subscribes to annotation and resource gather requests on EventBus; searches vectors for semantically similar passages
   - **Matcher** — subscribes to candidate search requests on EventBus
+  - **Smelter** — subscribes to resource/annotation events, chunks text, embeds, indexes into Qdrant
   - **CloneTokenManager** — subscribes to clone token operations on EventBus
 - **JobQueue** — background job processing queue + job status subscription
 - **6 annotation workers** — poll job queue for async AI tasks
@@ -98,12 +100,18 @@ graph TB
         BROWSER["Browser<br/>(read)"]
         GATHERER["Gatherer<br/>(context assembly)"]
         MATCHER["Matcher<br/>(search/link)"]
+        SMELTER["Smelter<br/>(embed)"]
         CTM["CloneTokenManager<br/>(clone)"]
         KB["Knowledge Base"]
+        VECTORS["Vector Store<br/>(Qdrant)"]
         STOWER -->|persist| KB
         BROWSER -->|query| KB
         GATHERER -->|query| KB
+        GATHERER -->|search| VECTORS
         MATCHER -->|query| KB
+        MATCHER -->|search| VECTORS
+        SMELTER -->|embed & index| VECTORS
+        SMELTER -->|read| KB
         CTM -->|query| KB
     end
 
@@ -111,12 +119,14 @@ graph TB
     BUS -->|"browse:resource-requested, browse:resources-requested<br/>browse:annotations-requested, browse:annotation-requested<br/>browse:events-requested, browse:annotation-history-requested<br/>browse:referenced-by-requested, browse:entity-types-requested<br/>browse:directory-requested"| BROWSER
     BUS -->|"gather:requested<br/>gather:resource-requested"| GATHERER
     BUS -->|"match:search-requested"| MATCHER
+    BUS -->|"yield:created, mark:created,<br/>mark:body-updated"| SMELTER
     BUS -->|"yield:clone-token-requested<br/>yield:clone-resource-requested<br/>yield:clone-create"| CTM
 
     STOWER -->|"yield:created, yield:updated, yield:moved<br/>mark:created, mark:deleted, mark:body-updated<br/>mark:entity-type-added, ..."| BUS
     BROWSER -->|"browse:resource-result, browse:resources-result<br/>browse:annotations-result, browse:annotation-result<br/>browse:events-result, browse:annotation-history-result<br/>browse:referenced-by-result, browse:entity-types-result<br/>browse:directory-result"| BUS
     GATHERER -->|"gather:complete, gather:failed<br/>gather:resource-complete, gather:resource-failed"| BUS
     MATCHER -->|"match:search-results, match:search-failed"| BUS
+    SMELTER -->|"embedding:computed,<br/>embedding:deleted"| BUS
     CTM -->|"yield:clone-token-generated<br/>yield:clone-resource-result<br/>yield:clone-created"| BUS
 
     classDef bus fill:#e8a838,stroke:#b07818,stroke-width:3px,color:#000,font-weight:bold
@@ -125,8 +135,10 @@ graph TB
     classDef caller fill:#4a90a4,stroke:#2c5f7a,stroke-width:2px,color:#fff
 
     class BUS bus
-    class STOWER,BROWSER,GATHERER,MATCHER,CTM actor
+    classDef vectorstore fill:#6b8e9d,stroke:#4a6a7a,stroke-width:2px,color:#fff
+    class STOWER,BROWSER,GATHERER,MATCHER,SMELTER,CTM actor
     class KB kb
+    class VECTORS vectorstore
     class Routes,Workers,EBC caller
 ```
 
@@ -134,7 +146,7 @@ graph TB
 
 The **Knowledge System** binds the Knowledge Base to its actors. Nothing outside the Knowledge System reads or writes the Knowledge Base directly.
 
-The **Knowledge Base** is an inert store — it has no intelligence, no goals, no decisions. It groups five subsystems:
+The **Knowledge Base** is an inert store — it has no intelligence, no goals, no decisions. It groups five core subsystems and two optional ones:
 
 | Store | Implementation | Purpose |
 |-------|---------------|---------|
@@ -143,12 +155,15 @@ The **Knowledge Base** is an inert store — it has no intelligence, no goals, n
 | **Content Store** | `WorkingTreeStore` | Working-tree files addressed by URI |
 | **Graph** | `GraphDatabase` | Eventually consistent relationship projection |
 | **Graph Consumer** | `GraphDBConsumer` | Event-to-graph synchronization pipeline |
+| **Vectors** *(optional)* | `VectorStore` | Semantic vector index (Qdrant + memory) via `@semiont/vectors` |
+| **Smelter** *(optional)* | `Smelter` | Embedding pipeline actor (chunk, embed, index) |
 
 ```typescript
 import { createKnowledgeBase } from '@semiont/make-meaning';
 
 const kb = await createKnowledgeBase(eventStore, project, graphDb, logger);
 // kb.eventStore, kb.views, kb.content, kb.graph, kb.graphConsumer
+// kb.vectors (optional), kb.smelter (optional)
 ```
 
 ### EventBus Ownership
@@ -184,8 +199,9 @@ The EventBus is created by the backend (or script) and passed into `startMakeMea
 
 - `Stower` — Write gateway actor
 - `Browser` — Read actor (all KB queries, directory listings merged with KB metadata)
-- `Gatherer` — Context assembly actor (annotation and resource gather flows)
+- `Gatherer` — Context assembly actor (annotation and resource gather flows; vector semantic search)
 - `Matcher` — Search/link actor (context-driven candidate search with structural + semantic scoring)
+- `Smelter` — Embedding pipeline actor (chunk, embed, persist, index into vector store)
 - `CloneTokenManager` — Clone token lifecycle actor (yield domain)
 
 ### Operations
@@ -214,6 +230,7 @@ The EventBus is created by the backend (or script) and passed into `startMakeMea
 - **[@semiont/graph](../graph/)** — Graph database abstraction
 - **[@semiont/ontology](../ontology/)** — Schema definitions for tags
 - **[@semiont/inference](../inference/)** — AI primitives (generateText)
+- **[@semiont/vectors](../vectors/)** — Vector store abstraction (Qdrant + memory) and embedding providers (Voyage, Ollama)
 - **[@semiont/jobs](../jobs/)** — Job queue and annotation workers
 
 ## Testing

package/dist/index.d.ts

@@ -1,10 +1,11 @@
 import { JobQueue, ReferenceAnnotationWorker, GenerationWorker, HighlightAnnotationWorker, AssessmentAnnotationWorker, CommentAnnotationWorker, TagAnnotationWorker } from '@semiont/jobs';
 import { SemiontProject } from '@semiont/core/node';
-import { GraphServiceConfig, Logger, StoredEvent, ResourceId, components, EventBus, AnnotationId, UserId, CreationMethod, ResourceAnnotations, AnnotationCategory, GraphPath, GraphConnection } from '@semiont/core';
+import { GraphServiceConfig, VectorsServiceConfig, EmbeddingServiceConfig, Logger, StoredEvent, ResourceId, EventBus, components, AnnotationId, UserId, CreationMethod, ResourceAnnotations, AnnotationCategory, GraphPath, GraphConnection } from '@semiont/core';
 export { AssembledAnnotation, applyBodyOperations, assembleAnnotation } from '@semiont/core';
 import { EventStore, ViewStorage } from '@semiont/event-sourcing';
 import { WorkingTreeStore } from '@semiont/content';
 import { GraphDatabase } from '@semiont/graph';
+import { VectorStore, EmbeddingProvider, ChunkingConfig } from '@semiont/vectors';
 import { InferenceClient } from '@semiont/inference';
 import { Writable, Readable } from 'node:stream';
 
@@ -44,6 +45,8 @@ interface WorkerInferenceConfig {
 interface MakeMeaningConfig {
     services: {
         graph?: GraphServiceConfig;
+        vectors?: VectorsServiceConfig;
+        embedding?: EmbeddingServiceConfig;
     };
     /** Per-actor inference config */
     actors?: ActorInferenceConfig;
@@ -143,17 +146,84 @@ declare class GraphDBConsumer {
     shutdown(): Promise<void>;
 }
 
+/**
+ * Smelter Actor
+ *
+ * Takes raw content, refines it into embedding vectors, persists them as events,
+ * and indexes them into the vector store. Peer to the Graph Consumer.
+ *
+ * Pipeline:
+ *   1. Subscribe to resource and annotation events from the EventStore
+ *   2. Chunk resource text into overlapping passages
+ *   3. Embed each chunk via the configured EmbeddingProvider
+ *   4. Emit embedding:computed events on the EventBus (persisted by Stower)
+ *   5. Index vectors into the VectorStore (Qdrant) for fast similarity search
+ *
+ * Uses the same burst-buffer RxJS pipeline as GraphDBConsumer.
+ */
+
+declare class Smelter {
+    private eventStore;
+    private eventBus;
+    private vectorStore;
+    private embeddingProvider;
+    private contentStore;
+    private static readonly SMELTER_RELEVANT_EVENTS;
+    private static readonly BURST_WINDOW_MS;
+    private static readonly MAX_BATCH_SIZE;
+    private static readonly IDLE_TIMEOUT_MS;
+    private _globalSubscription;
+    private eventSubject;
+    private pipelineSubscription;
+    private readonly logger;
+    private readonly chunkingConfig;
+    constructor(eventStore: EventStore, eventBus: EventBus, vectorStore: VectorStore, embeddingProvider: EmbeddingProvider, contentStore: WorkingTreeStore, logger: Logger, chunkingConfig?: ChunkingConfig);
+    initialize(): Promise<void>;
+    stop(): Promise<void>;
+    /**
+     * Rebuild the vector store from persisted embedding events in the event log.
+     * Reads all embedding.computed / embedding.deleted events and replays them.
+     * Bypasses the live pipeline — reads directly from the event store.
+     */
+    rebuildAll(): Promise<void>;
+    private processBatch;
+    /**
+     * Batch-optimized processing for consecutive events of the same type.
+     * Collects all texts across events, embeds in a single embedBatch() call,
+     * then distributes results back to their respective resources/annotations.
+     */
+    private applyBatchByType;
+    /**
+     * Batch-embed chunks from multiple resource.created events in a single
+     * embedBatch() call, then emit events and index per resource.
+     */
+    private batchResourceCreated;
+    /**
+     * Batch-embed exact texts from multiple annotation.added events in a
+     * single embedBatch() call, then emit events and index per annotation.
+     */
+    private batchAnnotationAdded;
+    private safeProcessEvent;
+    private processEvent;
+    private handleResourceCreated;
+    private handleResourceArchived;
+    private handleAnnotationAdded;
+    private handleAnnotationRemoved;
+}
+
 /**
  * Knowledge Base
  *
  * The durable store that records what intelligent actors decide.
- * Groups the five KB subsystems from ARCHITECTURE.md:
+ * Groups the KB subsystems from ARCHITECTURE.md:
  *
  * - Event Log (immutable append-only) — via EventStore
  * - Materialized Views (fast single-doc queries) — via ViewStorage
  * - Content Store (working-tree files, URI-addressed) — via WorkingTreeStore
  * - Graph (eventually consistent relationship projection) — via GraphDatabase
  * - Graph Consumer (event-to-graph projection) — via GraphDBConsumer
+ * - Vectors (semantic search) — via VectorStore (optional)
+ * - Smelter (event-to-vector projection) — via Smelter (optional)
  *
  * The Gatherer and Matcher are the only actors that read from these stores directly.
  */
@@ -164,9 +234,18 @@ interface KnowledgeBase {
     content: WorkingTreeStore;
     graph: GraphDatabase;
     graphConsumer: GraphDBConsumer;
+    vectors?: VectorStore;
+    smelter?: Smelter;
     projectionsDir: string;
 }
-declare function createKnowledgeBase(eventStore: EventStore, project: SemiontProject, graphDb: GraphDatabase, logger: Logger): Promise<KnowledgeBase>;
+interface CreateKnowledgeBaseOptions {
+    vectorStore?: VectorStore;
+    embeddingProvider?: EmbeddingProvider;
+    eventBus?: EventBus;
+    chunkingConfig?: ChunkingConfig;
+    skipRebuild?: boolean;
+}
+declare function createKnowledgeBase(eventStore: EventStore, project: SemiontProject, graphDb: GraphDatabase, logger: Logger, options?: CreateKnowledgeBaseOptions): Promise<KnowledgeBase>;
 
 /**
  * Stower Actor
@@ -198,6 +277,8 @@ declare function createKnowledgeBase(eventStore: EventStore, project: SemiontPro
  * - job:report-progress → job.progress
  * - job:complete       → job.completed
  * - job:fail           → job.failed
+ * - embedding:computed  → embedding.computed   (from Smelter)
+ * - embedding:deleted   → embedding.deleted    (from Smelter)
  */
 
 type ResourceDescriptor$3 = components['schemas']['ResourceDescriptor'];
@@ -226,6 +307,8 @@ declare class Stower {
     private handleJobReportProgress;
     private handleJobComplete;
     private handleJobFail;
+    private handleEmbeddingComputed;
+    private handleEmbeddingDeleted;
     stop(): Promise<void>;
 }
 
@@ -251,9 +334,10 @@ declare class Gatherer {
     private kb;
     private eventBus;
     private inferenceClient;
+    private embeddingProvider?;
     private subscriptions;
     private readonly logger;
-    constructor(kb: KnowledgeBase, eventBus: EventBus, inferenceClient: InferenceClient, logger: Logger);
+    constructor(kb: KnowledgeBase, eventBus: EventBus, inferenceClient: InferenceClient, logger: Logger, embeddingProvider?: EmbeddingProvider | undefined);
     initialize(): Promise<void>;
     private handleAnnotationGather;
     private handleResourceGather;
@@ -278,9 +362,10 @@ declare class Matcher {
     private kb;
     private eventBus;
     private inferenceClient;
+    private embeddingProvider?;
     private subscriptions;
     private readonly logger;
-    constructor(kb: KnowledgeBase, eventBus: EventBus, logger: Logger, inferenceClient: InferenceClient);
+    constructor(kb: KnowledgeBase, eventBus: EventBus, logger: Logger, inferenceClient: InferenceClient, embeddingProvider?: EmbeddingProvider | undefined);
     initialize(): Promise<void>;
     private handleSearch;
     /**
@@ -308,6 +393,11 @@ declare class Matcher {
      * @returns Map of resourceId → score (0-1)
      */
     private inferenceSemanticScore;
+    /**
+     * Search vectors for semantically similar resources.
+     * Returns empty array if vectors or embedding provider are not configured.
+     */
+    private searchVectors;
     stop(): Promise<void>;
 }
 
@@ -427,7 +517,9 @@ type Workers = {
     comment: CommentAnnotationWorker;
     tag: TagAnnotationWorker;
 };
-declare function startMakeMeaning(project: SemiontProject, config: MakeMeaningConfig, eventBus: EventBus, logger: Logger): Promise<MakeMeaningService>;
+declare function startMakeMeaning(project: SemiontProject, config: MakeMeaningConfig, eventBus: EventBus, logger: Logger, options?: {
+    skipRebuild?: boolean;
+}): Promise<MakeMeaningService>;
 
 /**
  * Entity Types Bootstrap Service
@@ -825,7 +917,7 @@ declare class AnnotationContext {
      * @returns Rich context for LLM processing
      * @throws Error if annotation or resource not found
      */
-    static buildLLMContext(annotationId: AnnotationId, resourceId: ResourceId, kb: KnowledgeBase, options?: BuildContextOptions, inferenceClient?: InferenceClient, logger?: Logger): Promise<AnnotationLLMContextResponse>;
+    static buildLLMContext(annotationId: AnnotationId, resourceId: ResourceId, kb: KnowledgeBase, options?: BuildContextOptions, inferenceClient?: InferenceClient, logger?: Logger, embeddingProvider?: EmbeddingProvider): Promise<AnnotationLLMContextResponse>;
     /**
      * Get resource annotations from view storage (fast path)
      * Throws if view missing
@@ -992,4 +1084,4 @@ declare function generateReferenceSuggestions(referenceTitle: string, client: In
 declare const PACKAGE_NAME = "@semiont/make-meaning";
 declare const VERSION = "0.1.0";
 
-export { AnnotationContext, AnnotationOperations, BACKUP_FORMAT, type BackupContentReader, type BackupEventStoreReader, type BackupExporterOptions, type BackupImportResult, type BackupImporterOptions, type BackupManifestHeader, type BackupStreamSummary, Browser, type BuildContextOptions, CloneTokenManager, type ContentBlobResolver, type CreateAnnotationResult, type CreateResourceInput, type CreateResourceResult, FORMAT_VERSION, Gatherer, GraphContext, type GraphEdge, type GraphNode, type GraphRepresentation, type KnowledgeBase, type KnowledgeSystem, LLMContext, type LLMContextOptions, type LinkedDataContentReader, type LinkedDataExporterOptions, type LinkedDataImportResult, type LinkedDataImporterOptions, type LinkedDataViewReader, type ListResourcesFilters, type MakeMeaningConfig, type MakeMeaningService, Matcher, PACKAGE_NAME, type ReplayStats, ResourceContext, ResourceOperations, Stower, type UpdateAnnotationBodyResult, type UpdateResourceInput, VERSION, bootstrapEntityTypes, createKnowledgeBase, exportBackup, exportLinkedData, generateReferenceSuggestions, generateResourceSummary, importBackup, importLinkedData, isBackupManifest, readEntityTypesProjection, resetBootstrap, startMakeMeaning, stopKnowledgeSystem, validateManifestVersion };
+export { AnnotationContext, AnnotationOperations, BACKUP_FORMAT, type BackupContentReader, type BackupEventStoreReader, type BackupExporterOptions, type BackupImportResult, type BackupImporterOptions, type BackupManifestHeader, type BackupStreamSummary, Browser, type BuildContextOptions, CloneTokenManager, type ContentBlobResolver, type CreateAnnotationResult, type CreateResourceInput, type CreateResourceResult, FORMAT_VERSION, Gatherer, GraphContext, type GraphEdge, type GraphNode, type GraphRepresentation, type KnowledgeBase, type KnowledgeSystem, LLMContext, type LLMContextOptions, type LinkedDataContentReader, type LinkedDataExporterOptions, type LinkedDataImportResult, type LinkedDataImporterOptions, type LinkedDataViewReader, type ListResourcesFilters, type MakeMeaningConfig, type MakeMeaningService, Matcher, PACKAGE_NAME, type ReplayStats, ResourceContext, ResourceOperations, Smelter, Stower, type UpdateAnnotationBodyResult, type UpdateResourceInput, VERSION, bootstrapEntityTypes, createKnowledgeBase, exportBackup, exportLinkedData, generateReferenceSuggestions, generateResourceSummary, importBackup, importLinkedData, isBackupManifest, readEntityTypesProjection, resetBootstrap, startMakeMeaning, stopKnowledgeSystem, validateManifestVersion };