@semiont/make-meaning 0.4.18 → 0.4.20

package/dist/index.d.ts

@@ -1,6 +1,6 @@
 import { JobQueue, ReferenceAnnotationWorker, GenerationWorker, HighlightAnnotationWorker, AssessmentAnnotationWorker, CommentAnnotationWorker, TagAnnotationWorker } from '@semiont/jobs';
 import { SemiontProject } from '@semiont/core/node';
-import { GraphServiceConfig, VectorsServiceConfig, EmbeddingServiceConfig, EventBus, Logger, StoredEvent, ResourceId, components, AnnotationId, UserId, CreationMethod, ResourceAnnotations, AnnotationCategory, GraphPath, GraphConnection } from '@semiont/core';
+import { GraphServiceConfig, VectorsServiceConfig, EmbeddingServiceConfig, EventBus, Logger, StoredEvent, ResourceId, AnnotationId, components, UserId, CreationMethod, ResourceAnnotations, AnnotationCategory, GraphPath, GraphConnection } from '@semiont/core';
 export { AssembledAnnotation, applyBodyOperations, assembleAnnotation } from '@semiont/core';
 import { EventStore, ViewStorage } from '@semiont/event-sourcing';
 import { WorkingTreeStore } from '@semiont/content';
@@ -72,6 +72,15 @@ interface MakeMeaningConfig {
  *   BURST_WINDOW_MS  = 50   — debounce window before flushing a batch
  *   MAX_BATCH_SIZE   = 500  — force flush to bound memory
  *   IDLE_TIMEOUT_MS  = 200  — silence before returning to passthrough
+ *
+ * ## Per-resource serialization
+ *
+ * `groupBy(resourceId) + concatMap(...)` is the stream-consumer flavor of
+ * per-resource serialization — the same invariant enforced by `Smelter`,
+ * `Gatherer`, and (in a different shape) `ViewManager`. See
+ * `packages/core/src/serialize-per-key.ts` for the shared primitive used
+ * by RPC-style services, and `.plans/PerResourceSerializer.md` for the
+ * broader design that would unify the two shapes.
  */
 
 declare class GraphDBConsumer {
@@ -147,28 +156,102 @@ declare class GraphDBConsumer {
     shutdown(): Promise<void>;
 }
 
+/**
+ * EmbeddingStore
+ *
+ * Durable file-based cache for pre-computed embedding vectors.
+ * Stored under .semiont/embeddings/ — committed to git alongside events,
+ * but overwritten in place rather than appended.
+ *
+ * File layout (same 4-hex Jump Consistent Hash sharding as events):
+ *
+ *   .semiont/embeddings/{ab}/{cd}/{resourceId}.jsonl
+ *     Line 0: { model, dimensions }                  ← model header
+ *     Line N: { chunkIndex, text, embedding[] }       ← one chunk per line
+ *
+ *   .semiont/embeddings/{ab}/{cd}/{annotationId}.json
+ *     { model, dimensions, resourceId, text, embedding[], motivation, entityTypes }
+ *
+ * rebuildAll() in Smelter reads these files and upserts into Qdrant without
+ * calling the embedding provider — unless the stored model doesn't match the
+ * configured provider, in which case the file is re-embedded and overwritten.
+ */
+
+interface StoredChunk {
+    chunkIndex: number;
+    text: string;
+    embedding: number[];
+}
+interface ResourceEmbeddingFile {
+    model: string;
+    dimensions: number;
+    chunks: StoredChunk[];
+}
+interface AnnotationEmbeddingFile {
+    model: string;
+    dimensions: number;
+    resourceId: string;
+    text: string;
+    embedding: number[];
+    motivation: string;
+    entityTypes: string[];
+}
+declare class EmbeddingStore {
+    private readonly project;
+    constructor(project: SemiontProject);
+    private resourceFilePath;
+    private annotationFilePath;
+    writeResourceChunks(resourceId: ResourceId, model: string, dimensions: number, chunks: StoredChunk[]): Promise<void>;
+    readResourceEmbeddings(resourceId: ResourceId): Promise<ResourceEmbeddingFile | null>;
+    deleteResourceEmbeddings(resourceId: ResourceId): Promise<void>;
+    writeAnnotationEmbedding(annotationId: AnnotationId, resourceId: ResourceId, model: string, dimensions: number, text: string, embedding: number[], motivation: string, entityTypes: string[]): Promise<void>;
+    readAnnotationEmbedding(annotationId: AnnotationId): Promise<AnnotationEmbeddingFile | null>;
+    deleteAnnotationEmbedding(annotationId: AnnotationId): Promise<void>;
+    /**
+     * Scan embeddings directory and return all resource IDs (from *.jsonl files).
+     */
+    getAllResourceIds(): Promise<string[]>;
+    /**
+     * Scan embeddings directory and return all annotation IDs (from *.json files).
+     */
+    getAllAnnotationIds(): Promise<string[]>;
+    private scanIds;
+}
+
 /**
  * Smelter Actor
  *
- * Takes raw content, refines it into embedding vectors, persists them as events,
- * and indexes them into the vector store. Peer to the Graph Consumer.
+ * Takes raw content, refines it into embedding vectors, persists them to the
+ * EmbeddingStore (.semiont/embeddings/), and indexes them into the VectorStore
+ * (Qdrant). Peer to the Graph Consumer.
  *
  * Pipeline:
  *   1. Subscribe to resource and annotation events from the EventStore
  *   2. Chunk resource text into overlapping passages
  *   3. Embed each chunk via the configured EmbeddingProvider
- *   4. Emit embedding:computed events on the EventBus (persisted by Stower)
+ *   4. Write vectors to EmbeddingStore (overwrite-in-place, git-durable)
  *   5. Index vectors into the VectorStore (Qdrant) for fast similarity search
  *
  * Uses the same burst-buffer RxJS pipeline as GraphDBConsumer.
+ *
+ * ## Per-resource serialization
+ *
+ * Smelter processes events strictly in order per resourceId via
+ * `groupBy(resourceId) + concatMap(...)`. This is the stream-consumer
+ * flavor of per-resource serialization — the same invariant enforced by
+ * `GraphDBConsumer`, `Gatherer`, and (in a different shape) `ViewManager`.
+ * See `packages/core/src/serialize-per-key.ts` for the shared primitive
+ * used by RPC-style services, and `.plans/PerResourceSerializer.md` for
+ * the broader design that would unify the two shapes.
  */
 
 declare class Smelter {
-    private eventStore;
     private eventBus;
     private vectorStore;
     private embeddingProvider;
     private contentStore;
+    private embeddingStore;
+    private viewStorage;
     private static readonly SMELTER_RELEVANT_EVENTS;
     private static readonly BURST_WINDOW_MS;
     private static readonly MAX_BATCH_SIZE;
@@ -178,35 +261,47 @@ declare class Smelter {
     private pipelineSubscription;
     private readonly logger;
     private readonly chunkingConfig;
-    constructor(eventStore: EventStore, eventBus: EventBus, vectorStore: VectorStore, embeddingProvider: EmbeddingProvider, contentStore: WorkingTreeStore, logger: Logger, chunkingConfig?: ChunkingConfig);
+    constructor(_eventStore: EventStore, eventBus: EventBus, vectorStore: VectorStore, embeddingProvider: EmbeddingProvider, contentStore: WorkingTreeStore, embeddingStore: EmbeddingStore, viewStorage: ViewStorage, logger: Logger, chunkingConfig?: ChunkingConfig);
     initialize(): Promise<void>;
     stop(): Promise<void>;
     /**
-     * Rebuild the vector store from persisted embedding events in the event log.
-     * Reads all embedding:computed / embedding:deleted events and replays them.
-     * Bypasses the live pipeline — reads directly from the event store.
+     * Rebuild the vector store from the EmbeddingStore (.semiont/embeddings/).
+     *
+     * For each stored file, checks whether the model matches the configured
+     * provider. On mismatch, re-embeds from the stored text and overwrites the
+     * file before upserting into Qdrant. On match, loads the stored vectors
+     * directly — no embedding provider calls needed.
      */
     rebuildAll(): Promise<void>;
     private processBatch;
     /**
      * Batch-optimized processing for consecutive events of the same type.
-     * Collects all texts across events, embeds in a single embedBatch() call,
-     * then distributes results back to their respective resources/annotations.
      */
     private applyBatchByType;
     /**
-     * Batch-embed chunks from multiple resource.created events in a single
-     * embedBatch() call, then emit events and index per resource.
+     * Batch-embed chunks from multiple yield:created events in a single
+     * embedBatch() call, then write to EmbeddingStore and index per resource.
      */
     private batchResourceCreated;
     /**
-     * Batch-embed exact texts from multiple annotation.added events in a
-     * single embedBatch() call, then emit events and index per annotation.
+     * Batch-embed exact texts from multiple mark:added events in a single
+     * embedBatch() call, then write to EmbeddingStore and index per annotation.
      */
     private batchAnnotationAdded;
     private safeProcessEvent;
     private processEvent;
     private handleResourceCreated;
+    /**
+     * Re-embed a resource whose content has changed in-place.
+     *
+     * Used by yield:updated and yield:representation-added handlers. Reads the
+     * current storageUri from the materialized view (which is updated before the
+     * EventBus fires), deletes stale Qdrant vectors, and overwrites the
+     * EmbeddingStore file with fresh chunks.
+     */
+    private reembedResource;
+    private handleResourceUpdated;
+    private handleRepresentationAdded;
     private handleResourceArchived;
     private handleAnnotationAdded;
     private handleAnnotationRemoved;
@@ -277,8 +372,6 @@ declare function createKnowledgeBase(eventStore: EventStore, project: SemiontPro
  * - job:report-progress → job.progress
  * - job:complete       → job.completed
  * - job:fail           → job.failed
- * - embedding:compute   → embedding.computed   (from Smelter)
- * - embedding:delete    → embedding.deleted    (from Smelter)
  */
 
 type ResourceDescriptor$3 = components['schemas']['ResourceDescriptor'];
@@ -307,8 +400,6 @@ declare class Stower {
     private handleJobReportProgress;
     private handleJobComplete;
     private handleJobFail;
-    private handleEmbeddingComputed;
-    private handleEmbeddingDeleted;
     stop(): Promise<void>;
 }
 
@@ -328,6 +419,15 @@ declare class Stower {
  * - gather:resource-requested — resource-level LLM context assembly
  *
  * RxJS pipeline uses groupBy(resourceId) + concatMap for per-resource isolation.
+ *
+ * ## Per-resource serialization
+ *
+ * `groupBy(resourceId) + concatMap(...)` is the stream-consumer flavor of
+ * per-resource serialization — the same invariant enforced by `Smelter`,
+ * `GraphDBConsumer`, and (in a different shape) `ViewManager`. See
+ * `packages/core/src/serialize-per-key.ts` for the shared primitive used
+ * by RPC-style services, and `.plans/PerResourceSerializer.md` for the
+ * broader design that would unify the two shapes.
  */
 
 declare class Gatherer {
@@ -581,8 +681,6 @@ interface BackupManifestHeader {
 interface BackupStreamSummary {
     stream: string;
     eventCount: number;
-    firstChecksum: string;
-    lastChecksum: string;
 }
 declare const LINKED_DATA_FORMAT: "semiont-linked-data";
 interface LinkedDataManifest {
@@ -687,7 +785,6 @@ interface BackupImporterOptions {
 interface BackupImportResult {
     manifest: BackupManifestHeader;
     stats: ReplayStats;
-    hashChainValid: boolean;
 }
 /**
  * Import a backup archive by replaying events through the EventBus.