@utaba/deep-memory-storage-cosmosdb 0.16.0 → 0.17.0

package/dist/index.d.cts

@@ -1,4 +1,4 @@
-import { StorageProvider, GraphTraversalProvider, EnsureSchemaResult, GraphTraversalCapabilities } from '@utaba/deep-memory/providers';
+import { StorageProvider, GraphTraversalProvider, EnsureSchemaResult, EntityReadOptions, GraphTraversalCapabilities } from '@utaba/deep-memory/providers';
 import { UsageSink, StorageRepositoryConfig, StoredRepository, RepositoryFilter, PaginatedResult, StoredRepositorySummary, RepositoryUpdate, DeleteProgressCallback, RepositoryStats, MemoryVocabulary, PaginationOptions, VocabularyChangeRecord, StoredEntity, StoredEntityUpdate, StorageFindQuery, StoredRelationship, RelationshipQueryOptions, StorageExploreOptions, StorageNeighborhood, StoragePathOptions, StoragePathResult, StorageTimelineOptions, StorageTimelineResult, ExportChunk, ImportChunk, BulkImportOptions, BulkImportResult, TraversalSpec, TraversalResult } from '@utaba/deep-memory/types';
 import gremlin from 'gremlin';
 
@@ -36,9 +36,17 @@ interface CosmosDbProviderConfig {
  */
 declare class CosmosDbProvider implements StorageProvider, GraphTraversalProvider {
     private readonly conn;
+    private readonly docClient;
     private readonly config;
     private readonly compiler;
     private readonly reportUsage;
+    /**
+     * Per-process vocabulary cache, keyed by repositoryId. Read lazily by
+     * `traverseImpl` via `getVocabularyCached`; invalidated on `saveVocabulary`.
+     * Each provider instance owns its own cache so isolated test providers do
+     * not share state.
+     */
+    private readonly vocabularyCache;
     constructor(config: CosmosDbProviderConfig);
     /**
      * Run the given operation inside an async-local usage scope. While the
@@ -68,6 +76,18 @@ declare class CosmosDbProvider implements StorageProvider, GraphTraversalProvide
     dispose(): Promise<void>;
     ensureSchema(): Promise<EnsureSchemaResult>;
     private ensureSchemaImpl;
+    /**
+     * Step E — verify the container's indexing policy covers every path that
+     * the SQL `findEntities` rewrite hits. The probe (2026-05-26) confirmed
+     * code-managed containers get the Cosmos default policy (everything
+     * indexed). This guard is for operator-facing protection: externally
+     * provisioned containers (ARM/Bicep) can have `excludedPaths` set, which
+     * would force `findEntities` to scan rather than index-lookup.
+     *
+     * Single GET against the colls resource, minimal RU. Never fails
+     * `ensureSchema` — a diagnostic must not break provisioning.
+     */
+    private runIndexingPolicyDiagnostic;
     /** Derive the REST endpoint from config — either explicit or from the Gremlin endpoint host. */
     private getRestEndpoint;
     createRepository(config: StorageRepositoryConfig): Promise<StoredRepository>;
@@ -81,24 +101,49 @@ declare class CosmosDbProvider implements StorageProvider, GraphTraversalProvide
     }>;
     getRepositoryStats(repositoryId: string): Promise<RepositoryStats>;
     getVocabulary(repositoryId: string): Promise<MemoryVocabulary>;
+    /**
+     * Cached vocabulary read used by traversal compilation. The vocabulary is
+     * compile-time context for the GremlinCompiler — it changes on the order of
+     * once per session, but the traversal hot path pays one round-trip per call
+     * to fetch it. The cache flips that to one round-trip per TTL window.
+     *
+     * Reads inside an active usage scope are still recorded if a fetch happens
+     * (cache miss); cache hits emit no round-trip and therefore no usage entry.
+     */
+    private getVocabularyCached;
+    /** Drop the cache entry for a repository — call after any vocabulary write. */
+    private invalidateVocabularyCache;
     saveVocabulary(repositoryId: string, vocabulary: MemoryVocabulary): Promise<void>;
     getVocabularyChangeLog(repositoryId: string, options?: PaginationOptions): Promise<PaginatedResult<VocabularyChangeRecord>>;
     createEntity(repositoryId: string, entity: StoredEntity): Promise<StoredEntity>;
-    getEntity(repositoryId: string, entityId: string): Promise<StoredEntity | null>;
-    getEntityBySlug(repositoryId: string, slug: string): Promise<StoredEntity | null>;
-    getEntities(repositoryId: string, entityIds: string[]): Promise<Map<string, StoredEntity>>;
+    getEntity(repositoryId: string, entityId: string, options?: EntityReadOptions): Promise<StoredEntity | null>;
+    getEntityBySlug(repositoryId: string, slug: string, options?: EntityReadOptions): Promise<StoredEntity | null>;
+    getEntities(repositoryId: string, entityIds: string[], options?: EntityReadOptions): Promise<Map<string, StoredEntity>>;
     updateEntity(repositoryId: string, entityId: string, updates: StoredEntityUpdate): Promise<StoredEntity>;
     deleteEntity(repositoryId: string, entityId: string): Promise<void>;
     deleteEntities(repositoryId: string, ids: string[]): Promise<{
         deleted: string[];
         notFound: string[];
     }>;
+    /**
+     * Single round-trip bulk-delete via the aggregate-side-effect pattern:
+     * collapses the per-chunk existence-check + drop into one Gremlin call.
+     *
+     *   g.V()...hasId(within(...)).has('entityType')
+     *     .aggregate('found').by('id')   // collects the ids that match
+     *     .drop()                         // drops the vertices (and cascaded edges)
+     *     .cap('found')                   // emits the bucket as the single result
+     *
+     * The bucket is always emitted as a single list item — empty when nothing
+     * matched (probe-verified 2026-05-25, local-tests/phase7-shape-probe.mjs).
+     * `notFound` is derived client-side as the set difference.
+     */
     private deleteEntitiesImpl;
     deleteEntitiesByType(repositoryId: string, entityType: string): Promise<{
         deletedEntities: number;
-        deletedRelationships: number;
+        deletedRelationships: number | undefined;
     }>;
-    findEntities(repositoryId: string, query: StorageFindQuery): Promise<PaginatedResult<StoredEntity>>;
+    findEntities(repositoryId: string, query: StorageFindQuery, options?: EntityReadOptions): Promise<PaginatedResult<StoredEntity>>;
     createRelationship(repositoryId: string, relationship: StoredRelationship): Promise<StoredRelationship>;
     getRelationship(repositoryId: string, relationshipId: string): Promise<StoredRelationship | null>;
     getEntityRelationships(repositoryId: string, entityId: string, options?: RelationshipQueryOptions): Promise<PaginatedResult<StoredRelationship>>;
@@ -107,12 +152,69 @@ declare class CosmosDbProvider implements StorageProvider, GraphTraversalProvide
         deleted: string[];
         notFound: string[];
     }>;
+    /**
+     * Single round-trip bulk relationship delete via the aggregate-side-effect
+     * pattern: collapses the per-chunk existence-check + drop into one Gremlin
+     * call. Gremlin drop on edges is routed by the engine and the bucket gives
+     * back the exact ids that were actually dropped, so `notFound` can be
+     * derived client-side.
+     *
+     * Source-id partition routing is not exposed on this method (the public
+     * surface accepts only edge ids), so the lookup may fan out across
+     * partitions — see [docs/cosmosdb-gremlin-compatibility.md §`g.E().has`
+     * doesn't always push partition down]. Callers that already hold a
+     * StoredRelationship and want partition-scoped routing should add a
+     * dedicated method when the need is concrete.
+     */
     private deleteRelationshipsImpl;
     deleteRelationshipsByType(repositoryId: string, relationshipType: string): Promise<{
         deletedRelationships: number;
     }>;
     exploreNeighborhood(repositoryId: string, entityId: string, options: StorageExploreOptions): Promise<StorageNeighborhood>;
+    /**
+     * Compiler-model implementation of exploreNeighborhood.
+     *
+     * Strategy: for each depth `d` from 1 to `options.depth`, build a
+     * cumulative `TraversalSpec` with `d` steps and `returnMode: 'all'`, run
+     * it through {@link executeTraversal}, then walk one BFS layer client-side
+     * from the previous frontier using the returned edges.
+     *
+     * The server-side step direction is fixed to `'both'` (catches every edge
+     * in either direction). The directional + bidirectional filter is applied
+     * client-side per hop — i.e. `direction: 'outbound'` includes inbound edges
+     * where `bidirectional` is true. The CosmosDB Gremlin compiler does not
+     * natively express the `union(outE, inE.has(bidirectional))` shape that
+     * would push this filter server-side, so it stays client-side; the
+     * observable contract (which edges count toward "outbound" given
+     * bidirectionality) is preserved.
+     *
+     * Round-trips per call: `options.depth` (one per layer). The previous BFS
+     * was `1 + fanout + fanout² + …` round-trips for the same depth.
+     */
+    private exploreNeighborhoodImpl;
     findPaths(repositoryId: string, sourceId: string, targetId: string, options: StoragePathOptions): Promise<StoragePathResult>;
+    /**
+     * Compiler-model implementation of findPaths.
+     *
+     * Strategy: build a `TraversalSpec` with a single `'both'` direction step
+     * in `repeat()` mode with `emitIntermediates: true` and `returnMode:
+     * 'path'`, run it once. The compiler always emits `.simplePath()` in path
+     * mode for cycle prevention, producing
+     * `.emit().repeat(bothE().otherV()).times(maxDepth).simplePath()
+     * .range(...).path().by(<v>).by(<e>)`, which yields paths of every length
+     * from 0 (the start vertex alone) to `maxDepth`. Live-probed against the
+     * Cosmos emulator 2026-05-25 — see local-tests/phase4-repeat-emit-probe.mjs.
+     *
+     * The pre-Phase-4 Cosmos BFS in (now-deleted) `packages/storage-cosmosdb/
+     * src/queries/traversal.ts` traversed edges with unconditional `bothE()`
+     * regardless of the `bidirectional` flag (path discovery is reachability,
+     * not semantic direction). Mirror that here by using step direction
+     * `'both'` and not applying any direction filter. The plan's §6
+     * observable-outputs rule requires preserving this.
+     *
+     * One round-trip total. The previous BFS was up to `1 + fanout + fanout² + …`.
+     */
+    private findPathsImpl;
     getTimeline(repositoryId: string, entityId: string, options: StorageTimelineOptions): Promise<StorageTimelineResult>;
     exportAll(repositoryId: string): AsyncIterable<ExportChunk>;
     importBulk(repositoryId: string, data: ImportChunk[], options?: BulkImportOptions): Promise<BulkImportResult>;
@@ -125,7 +227,34 @@ declare class CosmosDbProvider implements StorageProvider, GraphTraversalProvide
     private trackIterable;
     getCapabilities(): GraphTraversalCapabilities;
     traverse(repositoryId: string, spec: TraversalSpec): Promise<TraversalResult>;
-    private traverseImpl;
+    /**
+     * Compile + execute a TraversalSpec against this repository's partition.
+     *
+     * Internal entrypoint shared by `traverse` (the public surface) and the
+     * compiler-model rewrites of `exploreNeighborhood` / `findPaths`. Does NOT
+     * wrap in `track()` — the outer public method owns its own usage scope, and
+     * inner submits accumulate into that scope (the nested-scope guard in
+     * `track()` keeps nested public calls from emitting duplicate records).
+     */
+    private traverseInternal;
+    /**
+     * Lower-level traversal helper: compiles a spec, submits to Gremlin, and
+     * parses the rows into raw {@link StoredEntity} / {@link StoredRelationship}
+     * objects (no detail-level projection, no provenance stripping). Used by
+     * `traverseInternal` and by the storage-layer rewrites of
+     * `exploreNeighborhood` / `findPaths` that need the full stored shape to
+     * rebuild `StorageNeighborhood` / `StoragePathResult`.
+     *
+     * Returns a discriminated bag — only the fields relevant to `spec.returnMode`
+     * are populated:
+     * - `'terminal'`: `terminalEntities` (in row order, no dedup beyond what the
+     *    server emitted).
+     * - `'all'`: `allEntities` and `allRelationships`, already server-deduped.
+     * - `'path'`: `pathRows` plus the `entityMap` / `relationshipMap` lookup
+     *    tables and `pathRelFirstDirection` for the first-seen direction per
+     *    deduped edge id.
+     */
+    private executeTraversal;
     /**
      * Execute a raw Gremlin query with caller-supplied bindings.
      *

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { StorageProvider, GraphTraversalProvider, EnsureSchemaResult, GraphTraversalCapabilities } from '@utaba/deep-memory/providers';
+import { StorageProvider, GraphTraversalProvider, EnsureSchemaResult, EntityReadOptions, GraphTraversalCapabilities } from '@utaba/deep-memory/providers';
 import { UsageSink, StorageRepositoryConfig, StoredRepository, RepositoryFilter, PaginatedResult, StoredRepositorySummary, RepositoryUpdate, DeleteProgressCallback, RepositoryStats, MemoryVocabulary, PaginationOptions, VocabularyChangeRecord, StoredEntity, StoredEntityUpdate, StorageFindQuery, StoredRelationship, RelationshipQueryOptions, StorageExploreOptions, StorageNeighborhood, StoragePathOptions, StoragePathResult, StorageTimelineOptions, StorageTimelineResult, ExportChunk, ImportChunk, BulkImportOptions, BulkImportResult, TraversalSpec, TraversalResult } from '@utaba/deep-memory/types';
 import gremlin from 'gremlin';
 
@@ -36,9 +36,17 @@ interface CosmosDbProviderConfig {
  */
 declare class CosmosDbProvider implements StorageProvider, GraphTraversalProvider {
     private readonly conn;
+    private readonly docClient;
     private readonly config;
     private readonly compiler;
     private readonly reportUsage;
+    /**
+     * Per-process vocabulary cache, keyed by repositoryId. Read lazily by
+     * `traverseImpl` via `getVocabularyCached`; invalidated on `saveVocabulary`.
+     * Each provider instance owns its own cache so isolated test providers do
+     * not share state.
+     */
+    private readonly vocabularyCache;
     constructor(config: CosmosDbProviderConfig);
     /**
      * Run the given operation inside an async-local usage scope. While the
@@ -68,6 +76,18 @@ declare class CosmosDbProvider implements StorageProvider, GraphTraversalProvide
     dispose(): Promise<void>;
     ensureSchema(): Promise<EnsureSchemaResult>;
     private ensureSchemaImpl;
+    /**
+     * Step E — verify the container's indexing policy covers every path that
+     * the SQL `findEntities` rewrite hits. The probe (2026-05-26) confirmed
+     * code-managed containers get the Cosmos default policy (everything
+     * indexed). This guard is for operator-facing protection: externally
+     * provisioned containers (ARM/Bicep) can have `excludedPaths` set, which
+     * would force `findEntities` to scan rather than index-lookup.
+     *
+     * Single GET against the colls resource, minimal RU. Never fails
+     * `ensureSchema` — a diagnostic must not break provisioning.
+     */
+    private runIndexingPolicyDiagnostic;
     /** Derive the REST endpoint from config — either explicit or from the Gremlin endpoint host. */
     private getRestEndpoint;
     createRepository(config: StorageRepositoryConfig): Promise<StoredRepository>;
@@ -81,24 +101,49 @@ declare class CosmosDbProvider implements StorageProvider, GraphTraversalProvide
     }>;
     getRepositoryStats(repositoryId: string): Promise<RepositoryStats>;
     getVocabulary(repositoryId: string): Promise<MemoryVocabulary>;
+    /**
+     * Cached vocabulary read used by traversal compilation. The vocabulary is
+     * compile-time context for the GremlinCompiler — it changes on the order of
+     * once per session, but the traversal hot path pays one round-trip per call
+     * to fetch it. The cache flips that to one round-trip per TTL window.
+     *
+     * Reads inside an active usage scope are still recorded if a fetch happens
+     * (cache miss); cache hits emit no round-trip and therefore no usage entry.
+     */
+    private getVocabularyCached;
+    /** Drop the cache entry for a repository — call after any vocabulary write. */
+    private invalidateVocabularyCache;
     saveVocabulary(repositoryId: string, vocabulary: MemoryVocabulary): Promise<void>;
     getVocabularyChangeLog(repositoryId: string, options?: PaginationOptions): Promise<PaginatedResult<VocabularyChangeRecord>>;
     createEntity(repositoryId: string, entity: StoredEntity): Promise<StoredEntity>;
-    getEntity(repositoryId: string, entityId: string): Promise<StoredEntity | null>;
-    getEntityBySlug(repositoryId: string, slug: string): Promise<StoredEntity | null>;
-    getEntities(repositoryId: string, entityIds: string[]): Promise<Map<string, StoredEntity>>;
+    getEntity(repositoryId: string, entityId: string, options?: EntityReadOptions): Promise<StoredEntity | null>;
+    getEntityBySlug(repositoryId: string, slug: string, options?: EntityReadOptions): Promise<StoredEntity | null>;
+    getEntities(repositoryId: string, entityIds: string[], options?: EntityReadOptions): Promise<Map<string, StoredEntity>>;
     updateEntity(repositoryId: string, entityId: string, updates: StoredEntityUpdate): Promise<StoredEntity>;
     deleteEntity(repositoryId: string, entityId: string): Promise<void>;
     deleteEntities(repositoryId: string, ids: string[]): Promise<{
         deleted: string[];
         notFound: string[];
     }>;
+    /**
+     * Single round-trip bulk-delete via the aggregate-side-effect pattern:
+     * collapses the per-chunk existence-check + drop into one Gremlin call.
+     *
+     *   g.V()...hasId(within(...)).has('entityType')
+     *     .aggregate('found').by('id')   // collects the ids that match
+     *     .drop()                         // drops the vertices (and cascaded edges)
+     *     .cap('found')                   // emits the bucket as the single result
+     *
+     * The bucket is always emitted as a single list item — empty when nothing
+     * matched (probe-verified 2026-05-25, local-tests/phase7-shape-probe.mjs).
+     * `notFound` is derived client-side as the set difference.
+     */
     private deleteEntitiesImpl;
     deleteEntitiesByType(repositoryId: string, entityType: string): Promise<{
         deletedEntities: number;
-        deletedRelationships: number;
+        deletedRelationships: number | undefined;
     }>;
-    findEntities(repositoryId: string, query: StorageFindQuery): Promise<PaginatedResult<StoredEntity>>;
+    findEntities(repositoryId: string, query: StorageFindQuery, options?: EntityReadOptions): Promise<PaginatedResult<StoredEntity>>;
     createRelationship(repositoryId: string, relationship: StoredRelationship): Promise<StoredRelationship>;
     getRelationship(repositoryId: string, relationshipId: string): Promise<StoredRelationship | null>;
     getEntityRelationships(repositoryId: string, entityId: string, options?: RelationshipQueryOptions): Promise<PaginatedResult<StoredRelationship>>;
@@ -107,12 +152,69 @@ declare class CosmosDbProvider implements StorageProvider, GraphTraversalProvide
         deleted: string[];
         notFound: string[];
     }>;
+    /**
+     * Single round-trip bulk relationship delete via the aggregate-side-effect
+     * pattern: collapses the per-chunk existence-check + drop into one Gremlin
+     * call. Gremlin drop on edges is routed by the engine and the bucket gives
+     * back the exact ids that were actually dropped, so `notFound` can be
+     * derived client-side.
+     *
+     * Source-id partition routing is not exposed on this method (the public
+     * surface accepts only edge ids), so the lookup may fan out across
+     * partitions — see [docs/cosmosdb-gremlin-compatibility.md §`g.E().has`
+     * doesn't always push partition down]. Callers that already hold a
+     * StoredRelationship and want partition-scoped routing should add a
+     * dedicated method when the need is concrete.
+     */
     private deleteRelationshipsImpl;
     deleteRelationshipsByType(repositoryId: string, relationshipType: string): Promise<{
         deletedRelationships: number;
     }>;
     exploreNeighborhood(repositoryId: string, entityId: string, options: StorageExploreOptions): Promise<StorageNeighborhood>;
+    /**
+     * Compiler-model implementation of exploreNeighborhood.
+     *
+     * Strategy: for each depth `d` from 1 to `options.depth`, build a
+     * cumulative `TraversalSpec` with `d` steps and `returnMode: 'all'`, run
+     * it through {@link executeTraversal}, then walk one BFS layer client-side
+     * from the previous frontier using the returned edges.
+     *
+     * The server-side step direction is fixed to `'both'` (catches every edge
+     * in either direction). The directional + bidirectional filter is applied
+     * client-side per hop — i.e. `direction: 'outbound'` includes inbound edges
+     * where `bidirectional` is true. The CosmosDB Gremlin compiler does not
+     * natively express the `union(outE, inE.has(bidirectional))` shape that
+     * would push this filter server-side, so it stays client-side; the
+     * observable contract (which edges count toward "outbound" given
+     * bidirectionality) is preserved.
+     *
+     * Round-trips per call: `options.depth` (one per layer). The previous BFS
+     * was `1 + fanout + fanout² + …` round-trips for the same depth.
+     */
+    private exploreNeighborhoodImpl;
     findPaths(repositoryId: string, sourceId: string, targetId: string, options: StoragePathOptions): Promise<StoragePathResult>;
+    /**
+     * Compiler-model implementation of findPaths.
+     *
+     * Strategy: build a `TraversalSpec` with a single `'both'` direction step
+     * in `repeat()` mode with `emitIntermediates: true` and `returnMode:
+     * 'path'`, run it once. The compiler always emits `.simplePath()` in path
+     * mode for cycle prevention, producing
+     * `.emit().repeat(bothE().otherV()).times(maxDepth).simplePath()
+     * .range(...).path().by(<v>).by(<e>)`, which yields paths of every length
+     * from 0 (the start vertex alone) to `maxDepth`. Live-probed against the
+     * Cosmos emulator 2026-05-25 — see local-tests/phase4-repeat-emit-probe.mjs.
+     *
+     * The pre-Phase-4 Cosmos BFS in (now-deleted) `packages/storage-cosmosdb/
+     * src/queries/traversal.ts` traversed edges with unconditional `bothE()`
+     * regardless of the `bidirectional` flag (path discovery is reachability,
+     * not semantic direction). Mirror that here by using step direction
+     * `'both'` and not applying any direction filter. The plan's §6
+     * observable-outputs rule requires preserving this.
+     *
+     * One round-trip total. The previous BFS was up to `1 + fanout + fanout² + …`.
+     */
+    private findPathsImpl;
     getTimeline(repositoryId: string, entityId: string, options: StorageTimelineOptions): Promise<StorageTimelineResult>;
     exportAll(repositoryId: string): AsyncIterable<ExportChunk>;
     importBulk(repositoryId: string, data: ImportChunk[], options?: BulkImportOptions): Promise<BulkImportResult>;
@@ -125,7 +227,34 @@ declare class CosmosDbProvider implements StorageProvider, GraphTraversalProvide
     private trackIterable;
     getCapabilities(): GraphTraversalCapabilities;
     traverse(repositoryId: string, spec: TraversalSpec): Promise<TraversalResult>;
-    private traverseImpl;
+    /**
+     * Compile + execute a TraversalSpec against this repository's partition.
+     *
+     * Internal entrypoint shared by `traverse` (the public surface) and the
+     * compiler-model rewrites of `exploreNeighborhood` / `findPaths`. Does NOT
+     * wrap in `track()` — the outer public method owns its own usage scope, and
+     * inner submits accumulate into that scope (the nested-scope guard in
+     * `track()` keeps nested public calls from emitting duplicate records).
+     */
+    private traverseInternal;
+    /**
+     * Lower-level traversal helper: compiles a spec, submits to Gremlin, and
+     * parses the rows into raw {@link StoredEntity} / {@link StoredRelationship}
+     * objects (no detail-level projection, no provenance stripping). Used by
+     * `traverseInternal` and by the storage-layer rewrites of
+     * `exploreNeighborhood` / `findPaths` that need the full stored shape to
+     * rebuild `StorageNeighborhood` / `StoragePathResult`.
+     *
+     * Returns a discriminated bag — only the fields relevant to `spec.returnMode`
+     * are populated:
+     * - `'terminal'`: `terminalEntities` (in row order, no dedup beyond what the
+     *    server emitted).
+     * - `'all'`: `allEntities` and `allRelationships`, already server-deduped.
+     * - `'path'`: `pathRows` plus the `entityMap` / `relationshipMap` lookup
+     *    tables and `pathRelFirstDirection` for the first-seen direction per
+     *    deduped edge id.
+     */
+    private executeTraversal;
     /**
      * Execute a raw Gremlin query with caller-supplied bindings.
      *