@semiont/make-meaning 0.5.24 → 0.5.26

package/dist/index.d.ts

@@ -1,10 +1,10 @@
 import { JobQueue } from '@semiont/jobs';
 import { SemiontProject } from '@semiont/core/node';
-import { GraphServiceConfig, VectorsServiceConfig, EmbeddingServiceConfig, StateUnit, EventBus, Logger, ResourceId, ResourceDescriptor, AnnotationId, components, ITransport, BaseUrl, ConnectionState, SemiontError, UserDID, EventMap, IContentTransport, PutBinaryRequest, PutBinaryOptions, AccessToken, BusRequestPrimitive, StoredEvent, Annotation, UserId, GatheredContext, ResourceAnnotations, AnnotationCategory, GraphPath, GraphConnection } from '@semiont/core';
+import { GraphServiceConfig, VectorsServiceConfig, EmbeddingServiceConfig, StateUnit, EventBus, Logger, ResourceId, ResourceDescriptor, AnnotationId, components, ITransport, BaseUrl, ConnectionState, SemiontError, UserDID, EventMap, IContentTransport, PutBinaryRequest, PutBinaryOptions, ExtractionOutcome, AccessToken, BusRequestPrimitive, ChunkingConfig, StoredEvent, Annotation, UserId, GatheredContext, ResourceAnnotations, AnnotationCategory, GraphPath, GraphConnection } from '@semiont/core';
 import { EventStore, ViewStorage } from '@semiont/event-sourcing';
-import { WorkingTreeStore } from '@semiont/content';
+import { WorkingTreeStore, AnchoredTextStore } from '@semiont/content';
 import { GraphDatabase } from '@semiont/graph';
-import { VectorStore, EmbeddingProvider, ChunkingConfig } from '@semiont/vectors';
+import { VectorStore, EmbeddingProvider } from '@semiont/vectors';
 import { InferenceClient } from '@semiont/inference';
 import { BehaviorSubject, Observable } from 'rxjs';
 import { WorkerBus } from '@semiont/sdk';
@@ -176,6 +176,8 @@ interface KnowledgeBase {
     eventStore: EventStore;
     views: ViewStorage;
     content: WorkingTreeStore;
+    /** Derived coordinate maps for resources whose text had to be recovered. */
+    anchoredText: AnchoredTextStore;
     graph: GraphDatabase;
     weaveProgress: WeaveProgress;
     smeltProgress: SmeltProgress;
@@ -328,7 +330,7 @@ declare class Matcher {
      * Context-driven search: multi-source retrieval + composite scoring
      *
      * Retrieval sources:
-     * 1. Name match — graph.searchResources(searchTerm)
+     * 1. Name match — graph.listResources({ search: searchTerm })
      * 2. Entity type match — graph.listResources({ entityTypes })
      * 3. Graph neighborhood — connections from GatheredContext
      *
@@ -393,6 +395,21 @@ declare class Browser {
     private readonly logger;
     constructor(views: ViewStorage, kb: KnowledgeBase, eventBus: EventBus, project: SemiontProject, config: MakeMeaningConfig, logger: Logger);
     initialize(): Promise<void>;
+    /**
+     * Serve a resource's derived coordinate map (ANCHORED-TEXT-CACHE Lane 5).
+     *
+     * Read-your-writes on the same barrier `llm-context` uses for vectors: a
+     * caller may arrive before the Smelter has finished the resource it just
+     * uploaded, so a miss waits for that content generation to settle rather than
+     * reporting "no map" for a document that is merely still being read.
+     *
+     * **This path never invokes the engine.** The Smelter is the sole producer.
+     * A miss that survives the barrier answers `null`, and the caller degrades —
+     * for a PDF annotation that means geometry with no quoted text, which is what
+     * shipped before any of this existed. OCR in a request path is precisely what
+     * this design exists to avoid.
+     */
+    private handleAnchoredText;
     private handleBrowseResource;
     private handleBrowseResources;
     private handleBrowseAnnotations;
@@ -479,6 +496,17 @@ declare function stopKnowledgeSystem(ks: KnowledgeSystem): Promise<void>;
 interface MakeMeaningService {
     knowledgeSystem: KnowledgeSystem;
     jobQueue: JobQueue;
+    /**
+     * The one SemiontProject this backend is serving — the same instance the
+     * KnowledgeSystem, the job queue and the bus handlers were built from.
+     *
+     * Exposed so request handlers reach for it instead of improvising their own
+     * from `config._metadata`. Two of them used to do exactly that, casting an
+     * underscore-prefixed field twice per request to rebuild a project that
+     * already existed a few frames up — and a project rebuilt that way is
+     * missing everything the entry point supplied it with.
+     */
+    project: SemiontProject;
     stop: () => Promise<void>;
 }
 declare function startMakeMeaning(project: SemiontProject, config: MakeMeaningConfig, eventBus: EventBus, logger: Logger, options?: {
@@ -582,6 +610,36 @@ declare class LocalContentTransport implements IContentTransport {
     putBinary(_request: PutBinaryRequest, _options?: PutBinaryOptions): Promise<{
         resourceId: ResourceId;
     }>;
+    /**
+     * Store a derived coordinate map under the content checksum the producer
+     * read (PERSIST-ANCHORS decision A — see the interface doc for why the
+     * producer supplies the key). In local mode this is the same store the
+     * HTTP route writes to — one storage authority, reached the same way from
+     * every process (ANCHORED-TEXT-CACHE Lane 5).
+     */
+    putAnchoredText(checksum: string, outcome: ExtractionOutcome, _options?: {
+        auth?: AccessToken;
+    }): Promise<void>;
+    /** The stored outcome, or null when nothing has derived one — the common case. */
+    getAnchoredText(resourceId: ResourceId, _options?: {
+        auth?: AccessToken;
+    }): Promise<ExtractionOutcome | null>;
+    /**
+     * The cache-consult read (PERSIST-ANCHORS P2c), straight from the store —
+     * checksum-addressed, so no view resolution and no settle barrier: the
+     * caller holds the content identity already.
+     */
+    getAnchoredTextByChecksum(checksum: string, _options?: {
+        auth?: AccessToken;
+    }): Promise<ExtractionOutcome | null>;
+    /**
+     * The store's would-hit keys, straight from the store — planning data for
+     * the reconcile diff (PERSIST-ANCHORS P0), so no settle barrier applies:
+     * presence is being asked, not content at a moment.
+     */
+    listAnchoredTextKeys(_options?: {
+        auth?: AccessToken;
+    }): Promise<string[]>;
     getBinary(resourceId: ResourceId, _options?: {
         auth?: AccessToken;
     }): Promise<{
@@ -741,6 +799,8 @@ interface SmelterActorStateUnitOptions {
 }
 interface SmelterActorStateUnit extends StateUnit {
     events$: Observable<SmelterEvent>;
+    /** `smelt:rebuild-anchors` commands (PERSIST-ANCHORS P0) — see the command-channel note above. */
+    rebuildAnchors$: Observable<EventMap['smelt:rebuild-anchors']>;
     start(): void;
 }
 declare function createSmelterActorStateUnit(options: SmelterActorStateUnitOptions): SmelterActorStateUnit;
@@ -786,9 +846,18 @@ interface ReconcileSummary {
     resourcesEmbedded: number;
     /** Tag-only drift healed by payload restamps — never embedding calls (S13). */
     resourcesRestamped: number;
+    /** Lost anchored-text artifacts re-derived by re-extraction — never
+     *  embedding calls (PERSIST-ANCHORS P0, the third drift class). */
+    resourcesReanchored: number;
     resourceVectorsDeleted: number;
     annotationsEmbedded: number;
     annotationVectorsDeleted: number;
+    /** Live resources whose media type has an extractor — the coverage
+     *  denominator (SMELTER-MEDIA-TYPES extraction-coverage). */
+    resourcesEligible: number;
+    /** Resources with vectors after the drain — the coverage numerator;
+     *  eligible − indexed is the decline gap. */
+    resourcesIndexed: number;
 }
 type ReconcileState = {
     phase: 'pending';
@@ -818,13 +887,15 @@ interface SmelterTiming {
  * lanes and batch paths serve both kinds of input.
  */
 interface SmelterWorkItem {
-    type: 'smelt:embed' | 'smelt:restamp' | 'smelt:purge' | 'smelt:embed-annotation' | 'smelt:purge-annotation';
+    type: 'smelt:embed' | 'smelt:restamp' | 'smelt:reanchor' | 'smelt:purge' | 'smelt:embed-annotation' | 'smelt:purge-annotation';
     resourceId: string;
     payload: Record<string, unknown>;
 }
 type SmelterInput = SmelterEvent | SmelterWorkItem;
 declare class Smelter {
     private events$;
+    /** `smelt:rebuild-anchors` commands — a separate stream, never the event mailbox (see SmelterActorStateUnit). */
+    private rebuildAnchors$;
     private vectorStore;
     private embeddingProvider;
     private content;
@@ -837,12 +908,24 @@ declare class Smelter {
     private static readonly RECONCILE_WAVE;
     private eventSubject;
     private sourceSubscription;
+    private commandSubscription;
     private pipelineSubscription;
     private _eventsProcessed;
     private _reconcileState;
     private workDone;
+    private workFailed;
     private workWaiter;
-    constructor(events$: Observable<SmelterEvent>, vectorStore: VectorStore, embeddingProvider: EmbeddingProvider, content: IContentTransport, bus: BusRequestPrimitive, chunkingConfig: ChunkingConfig, timing: SmelterTiming, logger: Logger);
+    /**
+     * Serializes every planner drain (reconcile, anchored-text rebuilds):
+     * there is one waiter slot, and the weave:rebuild rule — rebuilds never
+     * interleave — applies to every unit here being a potential multi-second
+     * OCR pass.
+     */
+    private drainChain;
+    constructor(events$: Observable<SmelterEvent>, 
+    /** `smelt:rebuild-anchors` commands — a separate stream, never the event mailbox (see SmelterActorStateUnit). */
+    rebuildAnchors$: Observable<EventMap['smelt:rebuild-anchors']>, vectorStore: VectorStore, embeddingProvider: EmbeddingProvider, content: IContentTransport, bus: BusRequestPrimitive, chunkingConfig: ChunkingConfig, timing: SmelterTiming, logger: Logger);
+    private readonly anchoredStore;
     get eventsProcessed(): number;
     get reconcileState(): ReconcileState;
     initialize(): void;
@@ -870,6 +953,20 @@ declare class Smelter {
      * rides the next embed.
      */
     private restampResource;
+    /**
+     * Re-derive a lost anchored-text artifact from the resource's current
+     * bytes (PERSIST-ANCHORS P0, the third drift class). Extraction is the
+     * cost here — the vectors are already correct, so this NEVER calls the
+     * embedding provider, the vector store, or the settled signal: the index
+     * decision was already made and announced at its checksum; only the map
+     * is missing. Name the work for what it does (the S13 discipline).
+     *
+     * The publish is STRICT, unlike the embed path's best-effort side
+     * publish: here the artifact IS the job, so a store failure must throw —
+     * the pipeline logs and counts it, and the rebuild command's partial-
+     * failure accounting depends on that throw.
+     */
+    private reanchorResource;
     private handleResourcePurge;
     /**
      * Resolve a resource's embeddable text: bytes via the content transport,
@@ -944,8 +1041,44 @@ declare class Smelter {
      * completion. The pipeline ticks `noteWorkDone` for every consumed work
      * item (success or failure — failures are logged like any live event), so
      * each wave's waiter resolves exactly when its items have been processed.
+     *
+     * Serialized through `drainChain`: there is ONE waiter slot, and the
+     * planners that drain (reconcile, `smelt:rebuild-anchors`) must not
+     * interleave — a rebuild command arriving mid-reconcile waits its turn.
+     *
+     * @returns how many of THESE items failed — the rebuild command's
+     * partial-failure accounting (failure detail is in the logs).
      */
     private drain;
+    /**
+     * `smelt:rebuild-anchors` — the operator's explicit re-derivation of
+     * anchored-text artifacts (PERSIST-ANCHORS P0), shaped after
+     * `weave:rebuild`: optionally scoped, strictly serialized (concatMap on
+     * the command stream + the drain chain), correlated ok/failed replies,
+     * and partial completion FAILS — a rebuild that quietly skipped resources
+     * would present exactly like a document with no text, which is the #845
+     * failure mode wearing different clothes.
+     *
+     * Never destructive: nothing is deleted first, stale entries are simply
+     * overwritten (the W5-frames lesson — a rebuild that clears before it
+     * re-derives turns a partial failure into a loss). Re-anchoring makes
+     * zero embedding calls; work items ride the normal per-resource lanes,
+     * so a rebuild can never interleave with live processing of the same
+     * resource (S1/S2).
+     */
+    private rebuildAnchors;
+    /**
+     * Embeddable live resources, each with the catalog's claims: the primary
+     * representation's checksum (the bytes the smelter would read), the
+     * current entity-type set (the discriminator the stamps must carry), and
+     * whether the media type's extractor derives geometry (whether an
+     * anchored-text artifact should exist). Embeddable ⇔ an extractor exists
+     * for the media type's strategy — the same registry the live fetch
+     * resolves, and `yieldsGeometry` is declared on the extractor itself, so
+     * every gate here and the live fetch's behavior are twins by construction.
+     * Shared by `reconcile()` and the `smelt:rebuild-anchors` planner.
+     */
+    private classifyEmbeddable;
     /** Page through `browse:resources-requested` until the catalog is exhausted. */
     private listAllResources;
 }
@@ -1254,6 +1387,14 @@ declare class AnnotationOperations {
 interface ListResourcesFilters {
     search?: string;
     archived?: boolean;
+    entityType?: string;
+    offset?: number;
+    limit?: number;
+}
+interface ListResourcesResult {
+    resources: ResourceDescriptor[];
+    /** Size of the whole match set, not of the returned page. */
+    total: number;
 }
 declare class ResourceContext {
     /**
@@ -1261,17 +1402,20 @@ declare class ResourceContext {
      */
     static getResourceMetadata(resourceId: ResourceId, kb: KnowledgeBase): Promise<ResourceDescriptor | null>;
     /**
-     * List resources, optionally filtered.
+     * List resources, optionally filtered, as one page plus the size of the whole
+     * match set. Every filter is applied before pagination on both paths — a
+     * filter applied afterwards narrows the page rather than the match set, which
+     * is how a search scoped to an entity type can come back empty while hundreds
+     * of resources match.
      *
-     * When `search` is set, delegates to `kb.graph.searchResources`, which runs
-     * the name match in the graph engine instead of scanning every view in JS.
-     * The graph result is then narrowed by `archived` if requested.
+     * When `search` is set, the entire query — filtering, ordering and
+     * pagination — runs inside the graph engine.
      *
-     * When `search` is unset, falls back to scanning all materialized views.
-     * (TODO: also push the listing path through the graph for large KBs.)
+     * When `search` is unset, the materialized views answer instead. They are the
+     * barrier-stamped projection, so an unsearched listing is read-your-writes
+     * where the graph is only eventually consistent.
      */
-    static listResources(filters: ListResourcesFilters | undefined, kb: KnowledgeBase): Promise<ResourceDescriptor[]>;
-    private static sortByDateDesc;
+    static listResources(filters: ListResourcesFilters | undefined, kb: KnowledgeBase): Promise<ListResourcesResult>;
     /**
      * Add content previews to resources (for search results)
      * Retrieves and decodes the first 200 characters of each resource's primary representation
@@ -1407,11 +1551,6 @@ declare class GraphContext {
      * Requires graph traversal - must use graph database
      */
     static getResourceConnections(resourceId: ResourceId, kb: KnowledgeBase): Promise<GraphConnection[]>;
-    /**
-     * Search resources by name (cross-resource query)
-     * Requires full-text search - must use graph database
-     */
-    static searchResources(query: string, kb: KnowledgeBase, limit?: number): Promise<ResourceDescriptor[]>;
     /**
      * Build the unified knowledge graph for a resource's neighborhood:
      * resources AND annotations as typed nodes, typed/directional edges.
@@ -1487,4 +1626,4 @@ declare function generateResourceSummary(resourceName: string, content: string,
 declare function generateReferenceSuggestions(referenceTitle: string, client: InferenceClient, entityType?: string, currentContent?: string): Promise<string[] | null>;
 
 export { AnnotationContext, AnnotationOperations, BACKUP_FORMAT, Browser, CloneTokenManager, FORMAT_VERSION, Gatherer$1 as Gatherer, GraphContext, LLMContext, LocalContentTransport, LocalTransport, Matcher, ResourceContext, ResourceOperations, Smelter, Stower, asBusRequestPrimitive, bootstrapEntityTypes, createKnowledgeBase, createSmelterActorStateUnit, exportBackup, exportLinkedData, generateReferenceSuggestions, generateResourceSummary, importBackup, importLinkedData, isBackupManifest, readEntityTypesProjection, registerAnnotationAssemblyHandler, registerAnnotationLookupHandlers, registerBindUpdateBodyHandler, registerBusHandlers, registerJobCommandHandlers, startMakeMeaning, stopKnowledgeSystem, validateManifestVersion };
-export type { BackupContentReader, BackupEventStoreReader, BackupExporterOptions, BackupImportResult, BackupImporterOptions, BackupManifestHeader, BackupStreamSummary, BuildContextOptions, ContentBlobResolver, CreateAnnotationResult, CreateResourceInput, CreateResourceResult, KnowledgeBase, KnowledgeSystem, LLMContextOptions, LinkedDataContentReader, LinkedDataExporterOptions, LinkedDataImportResult, LinkedDataImporterOptions, LinkedDataViewReader, ListResourcesFilters, LocalTransportConfig, MakeMeaningConfig, MakeMeaningService, ReconcileState, ReconcileSummary, ReplayStats, SmelterActorStateUnit, SmelterActorStateUnitOptions, SmelterEvent, SmelterInput, SmelterTiming, SmelterWorkItem, UpdateAnnotationBodyResult };
+export type { BackupContentReader, BackupEventStoreReader, BackupExporterOptions, BackupImportResult, BackupImporterOptions, BackupManifestHeader, BackupStreamSummary, BuildContextOptions, ContentBlobResolver, CreateAnnotationResult, CreateResourceInput, CreateResourceResult, KnowledgeBase, KnowledgeSystem, LLMContextOptions, LinkedDataContentReader, LinkedDataExporterOptions, LinkedDataImportResult, LinkedDataImporterOptions, LinkedDataViewReader, ListResourcesFilters, ListResourcesResult, LocalTransportConfig, MakeMeaningConfig, MakeMeaningService, ReconcileState, ReconcileSummary, ReplayStats, SmelterActorStateUnit, SmelterActorStateUnitOptions, SmelterEvent, SmelterInput, SmelterTiming, SmelterWorkItem, UpdateAnnotationBodyResult };