@semiont/make-meaning 0.4.19 → 0.4.21

package/dist/index.d.ts

@@ -1,11 +1,11 @@
-import { JobQueue, ReferenceAnnotationWorker, GenerationWorker, HighlightAnnotationWorker, AssessmentAnnotationWorker, CommentAnnotationWorker, TagAnnotationWorker } from '@semiont/jobs';
+import { JobQueue } from '@semiont/jobs';
 import { SemiontProject } from '@semiont/core/node';
-import { GraphServiceConfig, VectorsServiceConfig, EmbeddingServiceConfig, EventBus, Logger, StoredEvent, ResourceId, AnnotationId, components, UserId, CreationMethod, ResourceAnnotations, AnnotationCategory, GraphPath, GraphConnection } from '@semiont/core';
+import { GraphServiceConfig, VectorsServiceConfig, EmbeddingServiceConfig, EventBus, Logger, StoredEvent, ResourceId, components, AnnotationId, UserId, CreationMethod, ResourceAnnotations, AnnotationCategory, GraphPath, GraphConnection } from '@semiont/core';
 export { AssembledAnnotation, applyBodyOperations, assembleAnnotation } from '@semiont/core';
 import { EventStore, ViewStorage } from '@semiont/event-sourcing';
 import { WorkingTreeStore } from '@semiont/content';
 import { GraphDatabase } from '@semiont/graph';
-import { VectorStore, EmbeddingProvider, ChunkingConfig } from '@semiont/vectors';
+import { VectorStore, EmbeddingProvider } from '@semiont/vectors';
 import { InferenceClient } from '@semiont/inference';
 import { Writable, Readable } from 'node:stream';
 
@@ -72,6 +72,14 @@ interface MakeMeaningConfig {
  *   BURST_WINDOW_MS  = 50   — debounce window before flushing a batch
  *   MAX_BATCH_SIZE   = 500  — force flush to bound memory
  *   IDLE_TIMEOUT_MS  = 200  — silence before returning to passthrough
+ *
+ * ## Per-resource serialization
+ *
+ * `groupBy(resourceId) + concatMap(...)` is the stream-consumer flavor of
+ * per-resource serialization — the same invariant enforced by `Smelter`,
+ * `Gatherer`, and (in a different shape) `ViewManager`. See
+ * `packages/core/src/serialize-per-key.ts` for the shared primitive used
+ * by RPC-style services.
  */
 
 declare class GraphDBConsumer {
@@ -147,147 +155,6 @@ declare class GraphDBConsumer {
     shutdown(): Promise<void>;
 }
 
-/**
- * EmbeddingStore
- *
- * Durable file-based cache for pre-computed embedding vectors.
- * Stored under .semiont/embeddings/ — committed to git alongside events,
- * but overwritten in place rather than appended.
- *
- * File layout (same 4-hex Jump Consistent Hash sharding as events):
- *
- *   .semiont/embeddings/{ab}/{cd}/{resourceId}.jsonl
- *     Line 0: { model, dimensions }                  ← model header
- *     Line N: { chunkIndex, text, embedding[] }       ← one chunk per line
- *
- *   .semiont/embeddings/{ab}/{cd}/{annotationId}.json
- *     { model, dimensions, resourceId, text, embedding[], motivation, entityTypes }
- *
- * rebuildAll() in Smelter reads these files and upserts into Qdrant without
- * calling the embedding provider — unless the stored model doesn't match the
- * configured provider, in which case the file is re-embedded and overwritten.
- */
-
-interface StoredChunk {
-    chunkIndex: number;
-    text: string;
-    embedding: number[];
-}
-interface ResourceEmbeddingFile {
-    model: string;
-    dimensions: number;
-    chunks: StoredChunk[];
-}
-interface AnnotationEmbeddingFile {
-    model: string;
-    dimensions: number;
-    resourceId: string;
-    text: string;
-    embedding: number[];
-    motivation: string;
-    entityTypes: string[];
-}
-declare class EmbeddingStore {
-    private readonly project;
-    constructor(project: SemiontProject);
-    private resourceFilePath;
-    private annotationFilePath;
-    writeResourceChunks(resourceId: ResourceId, model: string, dimensions: number, chunks: StoredChunk[]): Promise<void>;
-    readResourceEmbeddings(resourceId: ResourceId): Promise<ResourceEmbeddingFile | null>;
-    deleteResourceEmbeddings(resourceId: ResourceId): Promise<void>;
-    writeAnnotationEmbedding(annotationId: AnnotationId, resourceId: ResourceId, model: string, dimensions: number, text: string, embedding: number[], motivation: string, entityTypes: string[]): Promise<void>;
-    readAnnotationEmbedding(annotationId: AnnotationId): Promise<AnnotationEmbeddingFile | null>;
-    deleteAnnotationEmbedding(annotationId: AnnotationId): Promise<void>;
-    /**
-     * Scan embeddings directory and return all resource IDs (from *.jsonl files).
-     */
-    getAllResourceIds(): Promise<string[]>;
-    /**
-     * Scan embeddings directory and return all annotation IDs (from *.json files).
-     */
-    getAllAnnotationIds(): Promise<string[]>;
-    private scanIds;
-}
-
-/**
- * Smelter Actor
- *
- * Takes raw content, refines it into embedding vectors, persists them to the
- * EmbeddingStore (.semiont/embeddings/), and indexes them into the VectorStore
- * (Qdrant). Peer to the Graph Consumer.
- *
- * Pipeline:
- *   1. Subscribe to resource and annotation events from the EventStore
- *   2. Chunk resource text into overlapping passages
- *   3. Embed each chunk via the configured EmbeddingProvider
- *   4. Write vectors to EmbeddingStore (overwrite-in-place, git-durable)
- *   5. Index vectors into the VectorStore (Qdrant) for fast similarity search
- *
- * Uses the same burst-buffer RxJS pipeline as GraphDBConsumer.
- */
-
-declare class Smelter {
-    private eventBus;
-    private vectorStore;
-    private embeddingProvider;
-    private contentStore;
-    private embeddingStore;
-    private viewStorage;
-    private static readonly SMELTER_RELEVANT_EVENTS;
-    private static readonly BURST_WINDOW_MS;
-    private static readonly MAX_BATCH_SIZE;
-    private static readonly IDLE_TIMEOUT_MS;
-    private _globalSubscriptions;
-    private eventSubject;
-    private pipelineSubscription;
-    private readonly logger;
-    private readonly chunkingConfig;
-    constructor(_eventStore: EventStore, eventBus: EventBus, vectorStore: VectorStore, embeddingProvider: EmbeddingProvider, contentStore: WorkingTreeStore, embeddingStore: EmbeddingStore, viewStorage: ViewStorage, logger: Logger, chunkingConfig?: ChunkingConfig);
-    initialize(): Promise<void>;
-    stop(): Promise<void>;
-    /**
-     * Rebuild the vector store from the EmbeddingStore (.semiont/embeddings/).
-     *
-     * For each stored file, checks whether the model matches the configured
-     * provider. On mismatch, re-embeds from the stored text and overwrites the
-     * file before upserting into Qdrant. On match, loads the stored vectors
-     * directly — no embedding provider calls needed.
-     */
-    rebuildAll(): Promise<void>;
-    private processBatch;
-    /**
-     * Batch-optimized processing for consecutive events of the same type.
-     */
-    private applyBatchByType;
-    /**
-     * Batch-embed chunks from multiple yield:created events in a single
-     * embedBatch() call, then write to EmbeddingStore and index per resource.
-     */
-    private batchResourceCreated;
-    /**
-     * Batch-embed exact texts from multiple mark:added events in a single
-     * embedBatch() call, then write to EmbeddingStore and index per annotation.
-     */
-    private batchAnnotationAdded;
-    private safeProcessEvent;
-    private processEvent;
-    private handleResourceCreated;
-    /**
-     * Re-embed a resource whose content has changed in-place.
-     *
-     * Used by yield:updated and yield:representation-added handlers. Reads the
-     * current storageUri from the materialized view (which is updated before the
-     * EventBus fires), deletes stale Qdrant vectors, and overwrites the
-     * EmbeddingStore file with fresh chunks.
-     */
-    private reembedResource;
-    private handleResourceUpdated;
-    private handleRepresentationAdded;
-    private handleResourceArchived;
-    private handleAnnotationAdded;
-    private handleAnnotationRemoved;
-}
-
 /**
  * Knowledge Base
  *
@@ -299,10 +166,11 @@ declare class Smelter {
  * - Content Store (working-tree files, URI-addressed) — via WorkingTreeStore
  * - Graph (eventually consistent relationship projection) — via GraphDatabase
  * - Graph Consumer (event-to-graph projection) — via GraphDBConsumer
- * - Vectors (semantic search) — via VectorStore (optional)
- * - Smelter (event-to-vector projection) — via Smelter (optional)
+ * - Vectors (semantic search) — via VectorStore (optional, read-only)
  *
- * The Gatherer and Matcher are the only actors that read from these stores directly.
+ * The Smelter (event-to-vector projection) runs as an external actor
+ * via @semiont/jobs/smelter-main. It subscribes to domain events via
+ * the EventBus gateway, embeds content, and writes to Qdrant directly.
  */
 
 interface KnowledgeBase {
@@ -312,13 +180,10 @@ interface KnowledgeBase {
     graph: GraphDatabase;
     graphConsumer: GraphDBConsumer;
     vectors?: VectorStore;
-    smelter?: Smelter;
     projectionsDir: string;
 }
 interface CreateKnowledgeBaseOptions {
     vectorStore?: VectorStore;
-    embeddingProvider?: EmbeddingProvider;
-    chunkingConfig?: ChunkingConfig;
     skipRebuild?: boolean;
 }
 declare function createKnowledgeBase(eventStore: EventStore, project: SemiontProject, graphDb: GraphDatabase, eventBus: EventBus, logger: Logger, options?: CreateKnowledgeBaseOptions): Promise<KnowledgeBase>;
@@ -350,9 +215,14 @@ declare function createKnowledgeBase(eventStore: EventStore, project: SemiontPro
  * - mark:add-entity-type → entitytype.added                   → mark:entity-type-added / mark:entity-type-add-failed
  * - mark:update-entity-types → entitytag.added / entitytag.removed
  * - job:start          → job.started
- * - job:report-progress → job.progress
  * - job:complete       → job.completed
  * - job:fail           → job.failed
+ *
+ * Note: `job:report-progress` is intentionally NOT persisted. Progress
+ * events are ephemeral UI feedback and would clutter the event log
+ * (historical logs show ~3× as many progress entries as start+complete
+ * combined). UI consumers subscribe to the bus directly for live
+ * progress; the event log keeps only the durable lifecycle boundaries.
  */
 
 type ResourceDescriptor$3 = components['schemas']['ResourceDescriptor'];
@@ -378,7 +248,6 @@ declare class Stower {
     private handleAddEntityType;
     private handleUpdateEntityTypes;
     private handleJobStart;
-    private handleJobReportProgress;
     private handleJobComplete;
     private handleJobFail;
     stop(): Promise<void>;
@@ -400,6 +269,14 @@ declare class Stower {
  * - gather:resource-requested — resource-level LLM context assembly
  *
  * RxJS pipeline uses groupBy(resourceId) + concatMap for per-resource isolation.
+ *
+ * ## Per-resource serialization
+ *
+ * `groupBy(resourceId) + concatMap(...)` is the stream-consumer flavor of
+ * per-resource serialization — the same invariant enforced by `Smelter`,
+ * `GraphDBConsumer`, and (in a different shape) `ViewManager`. See
+ * `packages/core/src/serialize-per-key.ts` for the shared primitive used
+ * by RPC-style services.
  */
 
 declare class Gatherer {
@@ -578,17 +455,8 @@ declare function stopKnowledgeSystem(ks: KnowledgeSystem): Promise<void>;
 interface MakeMeaningService {
     knowledgeSystem: KnowledgeSystem;
     jobQueue: JobQueue;
-    workers: Workers;
     stop: () => Promise<void>;
 }
-type Workers = {
-    detection: ReferenceAnnotationWorker;
-    generation: GenerationWorker;
-    highlight: HighlightAnnotationWorker;
-    assessment: AssessmentAnnotationWorker;
-    comment: CommentAnnotationWorker;
-    tag: TagAnnotationWorker;
-};
 declare function startMakeMeaning(project: SemiontProject, config: MakeMeaningConfig, eventBus: EventBus, logger: Logger, options?: {
     skipRebuild?: boolean;
 }): Promise<MakeMeaningService>;
@@ -864,6 +732,7 @@ declare function importLinkedData(archive: Readable, options: LinkedDataImporter
  */
 
 type ContentFormat = components['schemas']['ContentFormat'];
+type Agent$1 = components['schemas']['Agent'];
 interface UpdateResourceInput {
     resourceId: ResourceId;
     userId: UserId;
@@ -881,6 +750,14 @@ interface CreateResourceInput {
     language?: string;
     entityTypes?: string[];
     creationMethod?: CreationMethod;
+    /** Provenance for AI-generated resources: source resource + annotation. */
+    generatedFrom?: {
+        resourceId?: string;
+        annotationId?: string;
+    };
+    generationPrompt?: string;
+    generator?: Agent$1 | Agent$1[];
+    isDraft?: boolean;
 }
 declare class ResourceOperations {
     /**
@@ -1171,4 +1048,4 @@ declare function generateReferenceSuggestions(referenceTitle: string, client: In
 declare const PACKAGE_NAME = "@semiont/make-meaning";
 declare const VERSION = "0.1.0";
 
-export { AnnotationContext, AnnotationOperations, BACKUP_FORMAT, type BackupContentReader, type BackupEventStoreReader, type BackupExporterOptions, type BackupImportResult, type BackupImporterOptions, type BackupManifestHeader, type BackupStreamSummary, Browser, type BuildContextOptions, CloneTokenManager, type ContentBlobResolver, type CreateAnnotationResult, type CreateResourceInput, type CreateResourceResult, FORMAT_VERSION, Gatherer, GraphContext, type GraphEdge, type GraphNode, type GraphRepresentation, type KnowledgeBase, type KnowledgeSystem, LLMContext, type LLMContextOptions, type LinkedDataContentReader, type LinkedDataExporterOptions, type LinkedDataImportResult, type LinkedDataImporterOptions, type LinkedDataViewReader, type ListResourcesFilters, type MakeMeaningConfig, type MakeMeaningService, Matcher, PACKAGE_NAME, type ReplayStats, ResourceContext, ResourceOperations, Smelter, Stower, type UpdateAnnotationBodyResult, type UpdateResourceInput, VERSION, bootstrapEntityTypes, createKnowledgeBase, exportBackup, exportLinkedData, generateReferenceSuggestions, generateResourceSummary, importBackup, importLinkedData, isBackupManifest, readEntityTypesProjection, startMakeMeaning, stopKnowledgeSystem, validateManifestVersion };
+export { AnnotationContext, AnnotationOperations, BACKUP_FORMAT, type BackupContentReader, type BackupEventStoreReader, type BackupExporterOptions, type BackupImportResult, type BackupImporterOptions, type BackupManifestHeader, type BackupStreamSummary, Browser, type BuildContextOptions, CloneTokenManager, type ContentBlobResolver, type CreateAnnotationResult, type CreateResourceInput, type CreateResourceResult, FORMAT_VERSION, Gatherer, GraphContext, type GraphEdge, type GraphNode, type GraphRepresentation, type KnowledgeBase, type KnowledgeSystem, LLMContext, type LLMContextOptions, type LinkedDataContentReader, type LinkedDataExporterOptions, type LinkedDataImportResult, type LinkedDataImporterOptions, type LinkedDataViewReader, type ListResourcesFilters, type MakeMeaningConfig, type MakeMeaningService, Matcher, PACKAGE_NAME, type ReplayStats, ResourceContext, ResourceOperations, Stower, type UpdateAnnotationBodyResult, type UpdateResourceInput, VERSION, bootstrapEntityTypes, createKnowledgeBase, exportBackup, exportLinkedData, generateReferenceSuggestions, generateResourceSummary, importBackup, importLinkedData, isBackupManifest, readEntityTypesProjection, startMakeMeaning, stopKnowledgeSystem, validateManifestVersion };